38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
87 !Subtarget.hasStdExtF()) {
88 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
89 "doesn't support the F instruction set extension (ignoring "
93 !Subtarget.hasStdExtD()) {
94 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
95 "doesn't support the D instruction set extension (ignoring "
119 if (Subtarget.hasStdExtZfhmin())
121 if (Subtarget.hasStdExtZfbfmin())
123 if (Subtarget.hasStdExtF())
125 if (Subtarget.hasStdExtD())
127 if (Subtarget.hasStdExtZhinxmin())
129 if (Subtarget.hasStdExtZfinx())
131 if (Subtarget.hasStdExtZdinx()) {
139 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
140 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
142 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
143 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
144 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
145 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
146 MVT::nxv4i64, MVT::nxv8i64};
148 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
149 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
151 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
152 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
154 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
156 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
158 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
159 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
160 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
161 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
162 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
163 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
164 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
165 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
166 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
167 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
168 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
171 auto addRegClassForRVV = [
this](
MVT VT) {
175 if (VT.getVectorMinNumElements() < MinElts)
178 unsigned Size = VT.getSizeInBits().getKnownMinValue();
181 RC = &RISCV::VRRegClass;
183 RC = &RISCV::VRM2RegClass;
185 RC = &RISCV::VRM4RegClass;
187 RC = &RISCV::VRM8RegClass;
194 for (
MVT VT : BoolVecVTs)
195 addRegClassForRVV(VT);
196 for (
MVT VT : IntVecVTs) {
197 if (VT.getVectorElementType() == MVT::i64 &&
200 addRegClassForRVV(VT);
204 for (
MVT VT : F16VecVTs)
205 addRegClassForRVV(VT);
208 for (
MVT VT : BF16VecVTs)
209 addRegClassForRVV(VT);
212 for (
MVT VT : F32VecVTs)
213 addRegClassForRVV(VT);
216 for (
MVT VT : F64VecVTs)
217 addRegClassForRVV(VT);
220 auto addRegClassForFixedVectors = [
this](
MVT VT) {
227 if (useRVVForFixedLengthVectorVT(VT))
228 addRegClassForFixedVectors(VT);
231 if (useRVVForFixedLengthVectorVT(VT))
232 addRegClassForFixedVectors(VT);
292 if (!(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
306 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
307 !(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()))
317 if (!Subtarget.hasStdExtZbb())
323 if (!Subtarget.hasStdExtZmmul()) {
325 }
else if (Subtarget.
is64Bit()) {
332 if (!Subtarget.hasStdExtM()) {
335 }
else if (Subtarget.
is64Bit()) {
337 {MVT::i8, MVT::i16, MVT::i32},
Custom);
347 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
350 }
else if (Subtarget.hasVendorXTHeadBb()) {
354 }
else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
363 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
364 Subtarget.hasVendorXTHeadBb())
368 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
376 if (Subtarget.hasStdExtZbb() ||
377 (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
382 if (Subtarget.hasStdExtZbb() ||
383 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
390 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
391 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
400 if (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()) {
402 }
else if (Subtarget.hasShortForwardBranchOpt()) {
405 }
else if (Subtarget.
is64Bit()) {
409 if (!Subtarget.hasVendorXTHeadCondMov())
412 static const unsigned FPLegalNodeTypes[] = {
413 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
414 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
426 static const unsigned FPOpToExpand[] = {
427 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW,
430 static const unsigned FPRndMode[] = {
431 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
434 static const unsigned ZfhminZfbfminPromoteOps[] = {
435 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM,
442 ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
445 if (Subtarget.hasStdExtZfbfmin()) {
468 if (Subtarget.hasStdExtZfa())
473 for (
auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
494 ISD::FNEARBYINT, MVT::f16,
495 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ?
Legal :
Promote);
497 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
498 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
537 if (Subtarget.hasStdExtZfa()) {
555 if (Subtarget.hasStdExtZfa()) {
632 if (Subtarget.hasStdExtZicbop()) {
636 if (Subtarget.hasStdExtA()) {
638 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
642 }
else if (Subtarget.hasForcedAtomics()) {
666 {MVT::i8, MVT::i16},
Custom);
677 static const unsigned IntegerVPOps[] = {
678 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
679 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
680 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
681 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
682 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
683 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
684 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
685 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
686 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
687 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
688 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
689 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
690 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
691 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
692 ISD::EXPERIMENTAL_VP_SPLAT};
694 static const unsigned FloatingPointVPOps[] = {
695 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
696 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
697 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
698 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
699 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
700 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
701 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
702 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
703 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
704 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
705 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
706 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
707 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
708 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
710 static const unsigned IntegerVecReduceOps[] = {
711 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
712 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
713 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
715 static const unsigned FloatingPointVecReduceOps[] = {
716 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
717 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
728 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
729 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
730 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
734 for (
MVT VT : BoolVecVTs) {
760 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
764 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
788 ISD::VP_TRUNCATE, ISD::VP_SETCC},
804 for (
MVT VT : IntVecVTs) {
815 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
865 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
866 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
890 if (Subtarget.hasStdExtZvkb()) {
898 if (Subtarget.hasStdExtZvbb()) {
902 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
908 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
917 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
923 for (
MVT VT : VecTupleVTs) {
944 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
947 ISD::FCEIL, ISD::FTRUNC, ISD::FFLOOR, ISD::FROUND,
954 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
961 ISD::VP_REDUCE_SEQ_FADD,
971 ISD::VP_FROUNDTOZERO,
977 ISD::VP_REDUCE_FMINIMUM,
978 ISD::VP_REDUCE_FMAXIMUM};
981 const auto SetCommonVFPActions = [&](
MVT VT) {
999 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1026 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1027 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1058 const auto SetCommonVFPExtLoadTruncStoreActions =
1060 for (
auto SmallVT : SmallerVTs) {
1068 const auto SetCommonPromoteToF32Actions = [&](
MVT VT) {
1077 ISD::VP_UINT_TO_FP},
1092 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1093 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1094 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1114 for (
MVT VT : F16VecVTs) {
1117 SetCommonVFPActions(VT);
1120 for (
MVT VT : F16VecVTs) {
1123 SetCommonPromoteToF32Actions(VT);
1128 for (
MVT VT : BF16VecVTs) {
1131 SetCommonPromoteToF32Actions(VT);
1136 for (
MVT VT : F32VecVTs) {
1139 SetCommonVFPActions(VT);
1140 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1141 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1146 for (
MVT VT : F64VecVTs) {
1149 SetCommonVFPActions(VT);
1150 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1151 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1152 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1158 if (!useRVVForFixedLengthVectorVT(VT))
1200 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1204 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1231 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1250 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT,
Custom);
1253 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1254 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1285 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1286 ISD::VECREDUCE_UMIN},
1291 if (Subtarget.hasStdExtZvkb())
1294 if (Subtarget.hasStdExtZvbb()) {
1316 if (!useRVVForFixedLengthVectorVT(VT))
1348 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1351 if (Subtarget.hasStdExtZfhmin()) {
1375 if (Subtarget.hasStdExtZfbfmin()) {
1394 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT,
Custom);
1397 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1403 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1408 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT},
1443 if (Subtarget.hasStdExtA())
1446 if (Subtarget.hasForcedAtomics()) {
1449 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1450 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1451 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1452 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1456 if (Subtarget.hasVendorXTHeadMemIdx()) {
1472 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
1497 if (Subtarget.hasStdExtZbb())
1500 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1504 if (Subtarget.hasStdExtZbkb())
1516 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1519 if (Subtarget.hasVendorXTHeadMemPair())
1548MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1553bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1555 bool IsScalable)
const {
1562 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1587 unsigned Intrinsic)
const {
1588 auto &
DL =
I.getDataLayout();
1590 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1591 bool IsUnitStrided,
bool UsePtrVal =
false) {
1596 Info.ptrVal =
I.getArgOperand(PtrOp);
1598 Info.fallbackAddressSpace =
1599 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1603 MemTy =
I.getArgOperand(0)->getType();
1606 MemTy =
I.getType();
1615 Info.align =
DL.getABITypeAlign(MemTy);
1624 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1628 switch (Intrinsic) {
1631 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1632 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1633 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1634 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1635 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1636 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1637 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1638 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1639 case Intrinsic::riscv_masked_cmpxchg_i32:
1641 Info.memVT = MVT::i32;
1642 Info.ptrVal =
I.getArgOperand(0);
1644 Info.align =
Align(4);
1648 case Intrinsic::riscv_seg2_load:
1649 case Intrinsic::riscv_seg3_load:
1650 case Intrinsic::riscv_seg4_load:
1651 case Intrinsic::riscv_seg5_load:
1652 case Intrinsic::riscv_seg6_load:
1653 case Intrinsic::riscv_seg7_load:
1654 case Intrinsic::riscv_seg8_load:
1655 return SetRVVLoadStoreInfo( 0,
false,
1657 case Intrinsic::riscv_seg2_store:
1658 case Intrinsic::riscv_seg3_store:
1659 case Intrinsic::riscv_seg4_store:
1660 case Intrinsic::riscv_seg5_store:
1661 case Intrinsic::riscv_seg6_store:
1662 case Intrinsic::riscv_seg7_store:
1663 case Intrinsic::riscv_seg8_store:
1665 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1668 case Intrinsic::riscv_vle:
1669 case Intrinsic::riscv_vle_mask:
1670 case Intrinsic::riscv_vleff:
1671 case Intrinsic::riscv_vleff_mask:
1672 return SetRVVLoadStoreInfo( 1,
1676 case Intrinsic::riscv_vse:
1677 case Intrinsic::riscv_vse_mask:
1678 return SetRVVLoadStoreInfo( 1,
1682 case Intrinsic::riscv_vlse:
1683 case Intrinsic::riscv_vlse_mask:
1684 case Intrinsic::riscv_vloxei:
1685 case Intrinsic::riscv_vloxei_mask:
1686 case Intrinsic::riscv_vluxei:
1687 case Intrinsic::riscv_vluxei_mask:
1688 return SetRVVLoadStoreInfo( 1,
1691 case Intrinsic::riscv_vsse:
1692 case Intrinsic::riscv_vsse_mask:
1693 case Intrinsic::riscv_vsoxei:
1694 case Intrinsic::riscv_vsoxei_mask:
1695 case Intrinsic::riscv_vsuxei:
1696 case Intrinsic::riscv_vsuxei_mask:
1697 return SetRVVLoadStoreInfo( 1,
1700 case Intrinsic::riscv_vlseg2:
1701 case Intrinsic::riscv_vlseg3:
1702 case Intrinsic::riscv_vlseg4:
1703 case Intrinsic::riscv_vlseg5:
1704 case Intrinsic::riscv_vlseg6:
1705 case Intrinsic::riscv_vlseg7:
1706 case Intrinsic::riscv_vlseg8:
1707 case Intrinsic::riscv_vlseg2ff:
1708 case Intrinsic::riscv_vlseg3ff:
1709 case Intrinsic::riscv_vlseg4ff:
1710 case Intrinsic::riscv_vlseg5ff:
1711 case Intrinsic::riscv_vlseg6ff:
1712 case Intrinsic::riscv_vlseg7ff:
1713 case Intrinsic::riscv_vlseg8ff:
1714 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1717 case Intrinsic::riscv_vlseg2_mask:
1718 case Intrinsic::riscv_vlseg3_mask:
1719 case Intrinsic::riscv_vlseg4_mask:
1720 case Intrinsic::riscv_vlseg5_mask:
1721 case Intrinsic::riscv_vlseg6_mask:
1722 case Intrinsic::riscv_vlseg7_mask:
1723 case Intrinsic::riscv_vlseg8_mask:
1724 case Intrinsic::riscv_vlseg2ff_mask:
1725 case Intrinsic::riscv_vlseg3ff_mask:
1726 case Intrinsic::riscv_vlseg4ff_mask:
1727 case Intrinsic::riscv_vlseg5ff_mask:
1728 case Intrinsic::riscv_vlseg6ff_mask:
1729 case Intrinsic::riscv_vlseg7ff_mask:
1730 case Intrinsic::riscv_vlseg8ff_mask:
1731 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1734 case Intrinsic::riscv_vlsseg2:
1735 case Intrinsic::riscv_vlsseg3:
1736 case Intrinsic::riscv_vlsseg4:
1737 case Intrinsic::riscv_vlsseg5:
1738 case Intrinsic::riscv_vlsseg6:
1739 case Intrinsic::riscv_vlsseg7:
1740 case Intrinsic::riscv_vlsseg8:
1741 case Intrinsic::riscv_vloxseg2:
1742 case Intrinsic::riscv_vloxseg3:
1743 case Intrinsic::riscv_vloxseg4:
1744 case Intrinsic::riscv_vloxseg5:
1745 case Intrinsic::riscv_vloxseg6:
1746 case Intrinsic::riscv_vloxseg7:
1747 case Intrinsic::riscv_vloxseg8:
1748 case Intrinsic::riscv_vluxseg2:
1749 case Intrinsic::riscv_vluxseg3:
1750 case Intrinsic::riscv_vluxseg4:
1751 case Intrinsic::riscv_vluxseg5:
1752 case Intrinsic::riscv_vluxseg6:
1753 case Intrinsic::riscv_vluxseg7:
1754 case Intrinsic::riscv_vluxseg8:
1755 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1758 case Intrinsic::riscv_vlsseg2_mask:
1759 case Intrinsic::riscv_vlsseg3_mask:
1760 case Intrinsic::riscv_vlsseg4_mask:
1761 case Intrinsic::riscv_vlsseg5_mask:
1762 case Intrinsic::riscv_vlsseg6_mask:
1763 case Intrinsic::riscv_vlsseg7_mask:
1764 case Intrinsic::riscv_vlsseg8_mask:
1765 case Intrinsic::riscv_vloxseg2_mask:
1766 case Intrinsic::riscv_vloxseg3_mask:
1767 case Intrinsic::riscv_vloxseg4_mask:
1768 case Intrinsic::riscv_vloxseg5_mask:
1769 case Intrinsic::riscv_vloxseg6_mask:
1770 case Intrinsic::riscv_vloxseg7_mask:
1771 case Intrinsic::riscv_vloxseg8_mask:
1772 case Intrinsic::riscv_vluxseg2_mask:
1773 case Intrinsic::riscv_vluxseg3_mask:
1774 case Intrinsic::riscv_vluxseg4_mask:
1775 case Intrinsic::riscv_vluxseg5_mask:
1776 case Intrinsic::riscv_vluxseg6_mask:
1777 case Intrinsic::riscv_vluxseg7_mask:
1778 case Intrinsic::riscv_vluxseg8_mask:
1779 return SetRVVLoadStoreInfo(
I.arg_size() - 6,
1782 case Intrinsic::riscv_vsseg2:
1783 case Intrinsic::riscv_vsseg3:
1784 case Intrinsic::riscv_vsseg4:
1785 case Intrinsic::riscv_vsseg5:
1786 case Intrinsic::riscv_vsseg6:
1787 case Intrinsic::riscv_vsseg7:
1788 case Intrinsic::riscv_vsseg8:
1789 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1792 case Intrinsic::riscv_vsseg2_mask:
1793 case Intrinsic::riscv_vsseg3_mask:
1794 case Intrinsic::riscv_vsseg4_mask:
1795 case Intrinsic::riscv_vsseg5_mask:
1796 case Intrinsic::riscv_vsseg6_mask:
1797 case Intrinsic::riscv_vsseg7_mask:
1798 case Intrinsic::riscv_vsseg8_mask:
1799 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1802 case Intrinsic::riscv_vssseg2:
1803 case Intrinsic::riscv_vssseg3:
1804 case Intrinsic::riscv_vssseg4:
1805 case Intrinsic::riscv_vssseg5:
1806 case Intrinsic::riscv_vssseg6:
1807 case Intrinsic::riscv_vssseg7:
1808 case Intrinsic::riscv_vssseg8:
1809 case Intrinsic::riscv_vsoxseg2:
1810 case Intrinsic::riscv_vsoxseg3:
1811 case Intrinsic::riscv_vsoxseg4:
1812 case Intrinsic::riscv_vsoxseg5:
1813 case Intrinsic::riscv_vsoxseg6:
1814 case Intrinsic::riscv_vsoxseg7:
1815 case Intrinsic::riscv_vsoxseg8:
1816 case Intrinsic::riscv_vsuxseg2:
1817 case Intrinsic::riscv_vsuxseg3:
1818 case Intrinsic::riscv_vsuxseg4:
1819 case Intrinsic::riscv_vsuxseg5:
1820 case Intrinsic::riscv_vsuxseg6:
1821 case Intrinsic::riscv_vsuxseg7:
1822 case Intrinsic::riscv_vsuxseg8:
1823 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1826 case Intrinsic::riscv_vssseg2_mask:
1827 case Intrinsic::riscv_vssseg3_mask:
1828 case Intrinsic::riscv_vssseg4_mask:
1829 case Intrinsic::riscv_vssseg5_mask:
1830 case Intrinsic::riscv_vssseg6_mask:
1831 case Intrinsic::riscv_vssseg7_mask:
1832 case Intrinsic::riscv_vssseg8_mask:
1833 case Intrinsic::riscv_vsoxseg2_mask:
1834 case Intrinsic::riscv_vsoxseg3_mask:
1835 case Intrinsic::riscv_vsoxseg4_mask:
1836 case Intrinsic::riscv_vsoxseg5_mask:
1837 case Intrinsic::riscv_vsoxseg6_mask:
1838 case Intrinsic::riscv_vsoxseg7_mask:
1839 case Intrinsic::riscv_vsoxseg8_mask:
1840 case Intrinsic::riscv_vsuxseg2_mask:
1841 case Intrinsic::riscv_vsuxseg3_mask:
1842 case Intrinsic::riscv_vsuxseg4_mask:
1843 case Intrinsic::riscv_vsuxseg5_mask:
1844 case Intrinsic::riscv_vsuxseg6_mask:
1845 case Intrinsic::riscv_vsuxseg7_mask:
1846 case Intrinsic::riscv_vsuxseg8_mask:
1847 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1903 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1905 return (SrcBits == 64 && DestBits == 32);
1916 return (SrcBits == 64 && DestBits == 32);
1927 if (SrcBits == DestBits * 2) {
1939 EVT MemVT = LD->getMemoryVT();
1940 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1950 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1958 return Subtarget.hasStdExtZbb() ||
1959 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1963 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1964 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1975 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1980 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1984 EVT VT =
Y.getValueType();
1990 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1996 if (Subtarget.hasStdExtZbs())
1997 return X.getValueType().isScalarInteger();
2000 if (Subtarget.hasVendorXTHeadBs())
2001 return C !=
nullptr;
2003 return C &&
C->getAPIntValue().ule(10);
2020 assert(Ty->isIntegerTy());
2022 unsigned BitSize = Ty->getIntegerBitWidth();
2023 if (BitSize > Subtarget.
getXLen())
2027 int64_t Val = Imm.getSExtValue();
2035 if (!Subtarget.enableUnalignedScalarMem())
2051 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2058 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2062 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2074 case Instruction::Add:
2075 case Instruction::Sub:
2076 case Instruction::Mul:
2077 case Instruction::And:
2078 case Instruction::Or:
2079 case Instruction::Xor:
2080 case Instruction::FAdd:
2081 case Instruction::FSub:
2082 case Instruction::FMul:
2083 case Instruction::FDiv:
2084 case Instruction::ICmp:
2085 case Instruction::FCmp:
2087 case Instruction::Shl:
2088 case Instruction::LShr:
2089 case Instruction::AShr:
2090 case Instruction::UDiv:
2091 case Instruction::SDiv:
2092 case Instruction::URem:
2093 case Instruction::SRem:
2094 case Instruction::Select:
2095 return Operand == 1;
2113 switch (
II->getIntrinsicID()) {
2114 case Intrinsic::fma:
2115 case Intrinsic::vp_fma:
2116 return Operand == 0 || Operand == 1;
2117 case Intrinsic::vp_shl:
2118 case Intrinsic::vp_lshr:
2119 case Intrinsic::vp_ashr:
2120 case Intrinsic::vp_udiv:
2121 case Intrinsic::vp_sdiv:
2122 case Intrinsic::vp_urem:
2123 case Intrinsic::vp_srem:
2124 case Intrinsic::ssub_sat:
2125 case Intrinsic::vp_ssub_sat:
2126 case Intrinsic::usub_sat:
2127 case Intrinsic::vp_usub_sat:
2128 return Operand == 1;
2130 case Intrinsic::vp_add:
2131 case Intrinsic::vp_mul:
2132 case Intrinsic::vp_and:
2133 case Intrinsic::vp_or:
2134 case Intrinsic::vp_xor:
2135 case Intrinsic::vp_fadd:
2136 case Intrinsic::vp_fmul:
2137 case Intrinsic::vp_icmp:
2138 case Intrinsic::vp_fcmp:
2139 case Intrinsic::smin:
2140 case Intrinsic::vp_smin:
2141 case Intrinsic::umin:
2142 case Intrinsic::vp_umin:
2143 case Intrinsic::smax:
2144 case Intrinsic::vp_smax:
2145 case Intrinsic::umax:
2146 case Intrinsic::vp_umax:
2147 case Intrinsic::sadd_sat:
2148 case Intrinsic::vp_sadd_sat:
2149 case Intrinsic::uadd_sat:
2150 case Intrinsic::vp_uadd_sat:
2152 case Intrinsic::vp_sub:
2153 case Intrinsic::vp_fsub:
2154 case Intrinsic::vp_fdiv:
2155 return Operand == 0 || Operand == 1;
2176 if (!Subtarget.sinkSplatOperands())
2179 for (
auto OpIdx :
enumerate(
I->operands())) {
2185 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2199 for (
Use &U :
Op->uses()) {
2245 if (!Subtarget.hasStdExtZfa())
2248 bool IsSupportedVT =
false;
2249 if (VT == MVT::f16) {
2250 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2251 }
else if (VT == MVT::f32) {
2252 IsSupportedVT =
true;
2253 }
else if (VT == MVT::f64) {
2254 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2255 IsSupportedVT =
true;
2265 bool ForCodeSize)
const {
2266 bool IsLegalVT =
false;
2269 else if (VT == MVT::f32)
2271 else if (VT == MVT::f64)
2273 else if (VT == MVT::bf16)
2274 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2286 return Imm.isZero();
2290 if (Imm.isNegZero())
2295 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2298 Subtarget.
getXLen(), Subtarget);
2304 unsigned Index)
const {
2317 if (EltVT == MVT::i1)
2330 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2337 if ((ResElts * 2) != SrcElts)
2378 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2380 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2394 LHS.getOpcode() ==
ISD::AND && LHS.hasOneUse() &&
2396 uint64_t Mask = LHS.getConstantOperandVal(1);
2401 ShAmt = LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2406 LHS = LHS.getOperand(0);
2415 int64_t
C = RHSC->getSExtValue();
2453 if (VT.
SimpleTy >= MVT::riscv_nxv1i8x2 &&
2454 VT.
SimpleTy <= MVT::riscv_nxv1i8x8)
2456 if (VT.
SimpleTy >= MVT::riscv_nxv2i8x2 &&
2457 VT.
SimpleTy <= MVT::riscv_nxv2i8x8)
2459 if (VT.
SimpleTy >= MVT::riscv_nxv4i8x2 &&
2460 VT.
SimpleTy <= MVT::riscv_nxv4i8x8)
2462 if (VT.
SimpleTy >= MVT::riscv_nxv8i8x2 &&
2463 VT.
SimpleTy <= MVT::riscv_nxv8i8x8)
2465 if (VT.
SimpleTy >= MVT::riscv_nxv16i8x2 &&
2466 VT.
SimpleTy <= MVT::riscv_nxv16i8x4)
2468 if (VT.
SimpleTy == MVT::riscv_nxv32i8x2)
2478 switch (KnownSize) {
2506 return RISCV::VRRegClassID;
2508 return RISCV::VRM2RegClassID;
2510 return RISCV::VRM4RegClassID;
2512 return RISCV::VRM8RegClassID;
2522 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2523 "Unexpected subreg numbering");
2524 return RISCV::sub_vrm1_0 +
Index;
2527 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2528 "Unexpected subreg numbering");
2529 return RISCV::sub_vrm2_0 +
Index;
2532 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2533 "Unexpected subreg numbering");
2534 return RISCV::sub_vrm4_0 +
Index;
2542 unsigned RegsPerField = std::max(1U, (
unsigned)VT.
getSizeInBits() /
2544 switch (RegsPerField) {
2547 return RISCV::VRN2M1RegClassID;
2549 return RISCV::VRN3M1RegClassID;
2551 return RISCV::VRN4M1RegClassID;
2553 return RISCV::VRN5M1RegClassID;
2555 return RISCV::VRN6M1RegClassID;
2557 return RISCV::VRN7M1RegClassID;
2559 return RISCV::VRN8M1RegClassID;
2563 return RISCV::VRN2M2RegClassID;
2565 return RISCV::VRN3M2RegClassID;
2567 return RISCV::VRN4M2RegClassID;
2571 return RISCV::VRN2M4RegClassID;
2579 return RISCV::VRRegClassID;
2588std::pair<unsigned, unsigned>
2590 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2592 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2593 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2594 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2595 "Register classes not ordered");
2602 if (VecRegClassID == SubRegClassID)
2603 return {RISCV::NoSubRegister, 0};
2606 "Only allow scalable vector subvector.");
2608 "Invalid vector tuple insert/extract for vector and subvector with "
2619 unsigned SubRegIdx = RISCV::NoSubRegister;
2620 for (
const unsigned RCID :
2621 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2622 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2626 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2631 return {SubRegIdx, InsertExtractIdx};
2636bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2665unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2672 "Unexpected opcode");
2674 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2676 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2679 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2753bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2754 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2763 "Expected legal fixed length vector!");
2766 unsigned MaxELen = Subtarget.
getELen();
2800 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2807 "Expected to convert into a scalable vector!");
2808 assert(V.getValueType().isFixedLengthVector() &&
2809 "Expected a fixed length vector operand!");
2819 "Expected to convert into a fixed length vector!");
2820 assert(V.getValueType().isScalableVector() &&
2821 "Expected a scalable vector operand!");
2844static std::pair<SDValue, SDValue>
2853static std::pair<SDValue, SDValue>
2866static std::pair<SDValue, SDValue>
2883std::pair<unsigned, unsigned>
2899 return std::make_pair(MinVLMAX, MaxVLMAX);
2911 EVT VT,
unsigned DefinedValues)
const {
2925 std::tie(LMul, Fractional) =
2928 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2930 Cost = (LMul * DLenFactor);
2973 Op.getValueType() == MVT::bf16) {
2974 bool IsStrict =
Op->isStrictFPOpcode();
2979 {Op.getOperand(0), Op.getOperand(1)});
2981 {
Op.getValueType(), MVT::Other},
2987 DAG.
getNode(
Op.getOpcode(),
DL, MVT::f32,
Op.getOperand(0)),
3002 MVT DstVT =
Op.getSimpleValueType();
3011 Src.getValueType() == MVT::bf16) {
3018 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3026 Opc,
DL, DstVT, Src,
3040 MVT SrcVT = Src.getSimpleValueType();
3046 if (SatVT != DstEltVT)
3049 MVT DstContainerVT = DstVT;
3050 MVT SrcContainerVT = SrcVT;
3056 "Expected same element count");
3065 {Src, Src, DAG.getCondCode(ISD::SETNE),
3066 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
3070 if (DstEltSize > (2 * SrcEltSize)) {
3076 MVT CvtContainerVT = DstContainerVT;
3077 MVT CvtEltVT = DstEltVT;
3078 if (SrcEltSize > (2 * DstEltSize)) {
3087 while (CvtContainerVT != DstContainerVT) {
3093 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
3100 Res, DAG.
getUNDEF(DstContainerVT), VL);
3110 bool IsStrict =
Op->isStrictFPOpcode();
3111 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3121 {
Op.getOperand(0), SrcVal});
3122 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
3123 {Ext.getValue(1), Ext.getValue(0)});
3126 DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f32, SrcVal));
3135 case ISD::FROUNDEVEN:
3137 case ISD::VP_FROUNDEVEN:
3141 case ISD::VP_FROUNDTOZERO:
3145 case ISD::VP_FFLOOR:
3153 case ISD::VP_FROUND:
3169 MVT VT =
Op.getSimpleValueType();
3176 MVT ContainerVT = VT;
3183 if (
Op->isVPOpcode()) {
3184 Mask =
Op.getOperand(1);
3188 VL =
Op.getOperand(2);
3210 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3224 switch (
Op.getOpcode()) {
3230 case ISD::VP_FFLOOR:
3232 case ISD::FROUNDEVEN:
3233 case ISD::VP_FROUND:
3234 case ISD::VP_FROUNDEVEN:
3235 case ISD::VP_FROUNDTOZERO: {
3250 case ISD::FNEARBYINT:
3251 case ISD::VP_FNEARBYINT:
3264 Src, Src, Mask, VL);
3279 MVT VT =
Op.getSimpleValueType();
3283 MVT ContainerVT = VT;
3295 MVT MaskVT = Mask.getSimpleValueType();
3298 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3299 DAG.getUNDEF(MaskVT), Mask, VL});
3303 {Chain, Src, Src, Src, Unorder, VL});
3304 Chain = Src.getValue(1);
3320 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3332 switch (
Op.getOpcode()) {
3343 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3349 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3353 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3362 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3363 Truncated, Mask, VL);
3369 Src, Src, Mask, VL);
3379 MVT VT =
Op.getSimpleValueType();
3407 MVT VT =
Op.getSimpleValueType();
3412 MVT ContainerVT = VT;
3455 "Unexpected vector MVT");
3483 return std::nullopt;
3501 unsigned EltSizeInBits) {
3504 return std::nullopt;
3505 bool IsInteger =
Op.getValueType().isInteger();
3507 std::optional<unsigned> SeqStepDenom;
3508 std::optional<APInt> SeqStepNum;
3509 std::optional<APInt> SeqAddend;
3510 std::optional<std::pair<APInt, unsigned>> PrevElt;
3511 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3516 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3517 for (
auto [Idx, Elt] :
enumerate(
Op->op_values())) {
3518 if (Elt.isUndef()) {
3519 Elts[Idx] = std::nullopt;
3523 Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3528 return std::nullopt;
3529 Elts[Idx] = *ExactInteger;
3533 for (
auto [Idx, Elt] :
enumerate(Elts)) {
3542 unsigned IdxDiff = Idx - PrevElt->second;
3543 APInt ValDiff = *Elt - PrevElt->first;
3551 int64_t Remainder = ValDiff.
srem(IdxDiff);
3556 return std::nullopt;
3557 ValDiff = ValDiff.
sdiv(IdxDiff);
3562 SeqStepNum = ValDiff;
3563 else if (ValDiff != SeqStepNum)
3564 return std::nullopt;
3567 SeqStepDenom = IdxDiff;
3568 else if (IdxDiff != *SeqStepDenom)
3569 return std::nullopt;
3573 if (!PrevElt || PrevElt->first != *Elt)
3574 PrevElt = std::make_pair(*Elt, Idx);
3578 if (!SeqStepNum || !SeqStepDenom)
3579 return std::nullopt;
3583 for (
auto [Idx, Elt] :
enumerate(Elts)) {
3587 (
APInt(EltSizeInBits, Idx) * *SeqStepNum).sdiv(*SeqStepDenom);
3589 APInt Addend = *Elt - ExpectedVal;
3592 else if (Addend != SeqAddend)
3593 return std::nullopt;
3596 assert(SeqAddend &&
"Must have an addend if we have a step");
3598 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3599 SeqAddend->getSExtValue()};
3620 MVT ContainerVT = VT;
3629 Idx, DAG.
getUNDEF(ContainerVT), Mask, VL);
3648 MVT VT =
Op.getSimpleValueType();
3660 unsigned MostCommonCount = 0;
3662 unsigned NumUndefElts =
3670 unsigned NumScalarLoads = 0;
3676 unsigned &
Count = ValueCounts[V];
3679 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3684 if (++
Count >= MostCommonCount) {
3686 MostCommonCount =
Count;
3690 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3691 unsigned NumDefElts = NumElts - NumUndefElts;
3692 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3698 ((MostCommonCount > DominantValueCountThreshold) ||
3711 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3712 LastOp != DominantValue) {
3721 Processed.insert(LastOp);
3726 const SDValue &V = OpIdx.value();
3727 if (V.isUndef() || !Processed.insert(V).second)
3729 if (ValueCounts[V] == 1) {
3738 return DAG.getConstant(V == V1, DL, XLenVT);
3754 MVT VT =
Op.getSimpleValueType();
3784 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3785 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3793 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3794 MVT IntegerViaVecVT =
3799 unsigned BitPos = 0, IntegerEltIdx = 0;
3802 for (
unsigned I = 0;
I < NumElts;) {
3804 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3805 Bits |= ((
uint64_t)BitValue << BitPos);
3811 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3812 if (NumViaIntegerBits <= 32)
3815 Elts[IntegerEltIdx] = Elt;
3824 if (NumElts < NumViaIntegerBits) {
3828 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3856 int64_t StepNumerator = SimpleVID->StepNumerator;
3857 unsigned StepDenominator = SimpleVID->StepDenominator;
3858 int64_t Addend = SimpleVID->Addend;
3860 assert(StepNumerator != 0 &&
"Invalid step");
3861 bool Negate =
false;
3862 int64_t SplatStepVal = StepNumerator;
3866 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3868 Negate = StepNumerator < 0;
3870 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3880 (SplatStepVal >= 0 || StepDenominator == 1) &&
isInt<5>(Addend)) {
3883 MVT VIDContainerVT =
3891 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3892 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3894 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3896 if (StepDenominator != 1) {
3901 if (Addend != 0 || Negate) {
3920 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3921 "Unexpected sequence type");
3925 unsigned ViaVecLen =
3932 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3933 const auto &SeqV = OpIdx.value();
3934 if (!SeqV.isUndef())
3936 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3942 if (ViaIntVT == MVT::i32)
3968 BV->getRepeatedSequence(Sequence) &&
3969 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3970 unsigned SeqLen = Sequence.size();
3972 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3973 ViaIntVT == MVT::i64) &&
3974 "Unexpected sequence type");
3979 const unsigned RequiredVL = NumElts / SeqLen;
3980 const unsigned ViaVecLen =
3982 NumElts : RequiredVL;
3985 unsigned EltIdx = 0;
3990 for (
const auto &SeqV : Sequence) {
3991 if (!SeqV.isUndef())
3993 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4000 if (ViaIntVT == MVT::i32)
4007 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
4008 "Unexpected bitcast sequence");
4012 MVT ViaContainerVT =
4019 if (ViaVecLen != RequiredVL)
4039 Source, DAG, Subtarget);
4059 return RISCV::PACKH;
4061 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
4076 MVT VT =
Op.getSimpleValueType();
4084 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4089 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
4103 if (Subtarget.hasStdExtZbkb())
4108 ElemDL, XLenVT,
A,
B),
4121 NewOperands.
reserve(NumElts / 2);
4123 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
4127 return DAG.
getNode(ISD::BITCAST,
DL, VT,
4133 MVT VT =
Op.getSimpleValueType();
4144 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) || EltVT == MVT::bf16) {
4149 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4150 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4237 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
4241 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4264 unsigned NumUndefElts =
4266 unsigned NumDefElts = NumElts - NumUndefElts;
4267 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4274 for (
unsigned i = 0; i < NumElts; i++) {
4276 if (i < NumElts / 2) {
4283 bool SelectMaskVal = (i < NumElts / 2);
4286 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4287 MaskVals.
size() == NumElts);
4322 unsigned UndefCount = 0;
4329 LinearBudget -= PerSlideCost;
4332 LinearBudget -= PerSlideCost;
4335 LinearBudget -= PerSlideCost;
4338 if (LinearBudget < 0)
4343 "Illegal type which will result in reserved encoding");
4368 Vec,
Offset, Mask, VL, Policy);
4381 Vec,
Offset, Mask, VL, Policy);
4396 if ((LoC >> 31) == HiC)
4418 return DAG.
getNode(ISD::BITCAST,
DL, VT, InterVec);
4426 Hi.getConstantOperandVal(1) == 31)
4445 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4457 bool HasPassthru = Passthru && !Passthru.
isUndef();
4458 if (!HasPassthru && !Passthru)
4465 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
4466 EltVT == MVT::bf16) {
4467 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4468 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
4471 Scalar = DAG.
getNode(ISD::BITCAST,
DL, MVT::i16, Scalar);
4473 Passthru = DAG.
getNode(ISD::BITCAST,
DL, IVT, Passthru);
4482 if (Scalar.getValueType().bitsLE(XLenVT)) {
4489 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4493 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4494 "Unexpected scalar for splat lowering!");
4518 SDValue ExtractedVal = Scalar.getOperand(0);
4523 MVT ExtractedContainerVT = ExtractedVT;
4526 DAG, ExtractedContainerVT, Subtarget);
4528 ExtractedVal, DAG, Subtarget);
4530 if (ExtractedContainerVT.
bitsLE(VT))
4545 if (!Scalar.getValueType().bitsLE(XLenVT))
4548 VT,
DL, DAG, Subtarget);
4556 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4582 if (Src != V2.getOperand(0))
4586 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4591 V2.getConstantOperandVal(1) != Mask.size())
4595 if (Mask[0] != 0 && Mask[0] != 1)
4600 for (
unsigned i = 1; i != Mask.size(); ++i)
4601 if (Mask[i] != Mask[i - 1] + 2)
4619 int Size = Mask.size();
4621 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4627 EvenSrc = StartIndexes[0];
4628 OddSrc = StartIndexes[1];
4631 if (EvenSrc != 0 && OddSrc != 0)
4641 int HalfNumElts = NumElts / 2;
4642 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4658 int Size = Mask.size();
4670 for (
int i = 0; i !=
Size; ++i) {
4676 int StartIdx = i - (M %
Size);
4684 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4687 Rotation = CandidateRotation;
4688 else if (Rotation != CandidateRotation)
4693 int MaskSrc = M <
Size ? 0 : 1;
4698 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4703 TargetSrc = MaskSrc;
4704 else if (TargetSrc != MaskSrc)
4711 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4712 assert((LoSrc >= 0 || HiSrc >= 0) &&
4713 "Failed to find a rotated input vector!");
4728 MVT ContainerVT = VT;
4731 assert(Src.getSimpleValueType().isFixedLengthVector());
4735 MVT SrcContainerVT =
4748 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4755 unsigned Shift = EvenElts ? 0 : EltBits;
4761 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4787 auto findNonEXTRACT_SUBVECTORParent =
4788 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4793 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4794 Offset += Parent.getConstantOperandVal(1);
4795 Parent = Parent.getOperand(0);
4797 return std::make_pair(Parent,
Offset);
4800 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4801 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4810 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4811 if (NewMask[i] == -1)
4814 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4815 NewMask[i] = NewMask[i] + V1IndexOffset;
4819 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4825 if (NewMask[0] <= 0)
4829 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4830 if (NewMask[i - 1] + 1 != NewMask[i])
4834 MVT SrcVT = Src.getSimpleValueType();
4865 int NumSubElts,
Index;
4870 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4871 SDValue InPlace = OpsSwapped ? V2 : V1;
4872 SDValue ToInsert = OpsSwapped ? V1 : V2;
4882 if (NumSubElts +
Index >= (
int)NumElts)
4896 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4908 bool OpsSwapped =
false;
4923 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4924 for (
unsigned i = S; i !=
E; ++i)
4925 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4931 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4932 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4935 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4937 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4942 auto OpCode = IsVSlidedown ?
4947 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4950 Splat, TrueMask, VL);
4961 MVT VecContainerVT = VecVT;
4978 MVT WideContainerVT = WideVT;
4984 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4997 }
else if (Subtarget.hasStdExtZvbb()) {
5002 OffsetVec, Passthru, Mask, VL);
5005 Interleaved, EvenV, Passthru, Mask, VL);
5013 Interleaved, OffsetVec, Passthru, Mask, VL);
5021 OddV, Passthru, Mask, VL);
5027 OddV, AllOnesVec, Passthru, Mask, VL);
5035 Interleaved, OddsMul, Passthru, Mask, VL);
5042 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
5088 if (ViaEltSize > NumElts)
5097 if (ViaEltSize > NumElts)
5103 if (ViaEltSize > NumElts)
5112 MVT &RotateVT,
unsigned &RotateAmt) {
5118 unsigned NumSubElts;
5120 NumElts, NumSubElts, RotateAmt))
5123 NumElts / NumSubElts);
5185 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
5188 OutMasks(VRegsPerSrc, {-1, {}});
5193 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
5194 int DstVecIdx = DstIdx / ElemsPerVReg;
5195 int DstSubIdx = DstIdx % ElemsPerVReg;
5196 int SrcIdx = Mask[DstIdx];
5197 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
5199 int SrcVecIdx = SrcIdx / ElemsPerVReg;
5200 int SrcSubIdx = SrcIdx % ElemsPerVReg;
5201 if (OutMasks[DstVecIdx].first == -1)
5202 OutMasks[DstVecIdx].first = SrcVecIdx;
5203 if (OutMasks[DstVecIdx].first != SrcVecIdx)
5209 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
5210 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
5224 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
5225 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
5226 if (SrcVecIdx == -1)
5228 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
5235 unsigned InsertIdx = DstVecIdx * NumOpElts;
5248 MVT VT =
Op.getSimpleValueType();
5263 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5287 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5288 V = V.getOperand(
Offset / OpElements);
5304 SDValue Ops[] = {Ld->getChain(),
5318 MVT SplatVT = ContainerVT;
5321 if (SVT == MVT::f16 && !Subtarget.hasStdExtZfh()) {
5330 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5331 Ld->getPointerInfo().getWithOffset(
Offset),
5332 Ld->getOriginalAlign(),
5336 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5337 Ld->getOriginalAlign(),
5338 Ld->getMemOperand()->getFlags());
5350 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5353 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5375 if (Subtarget.hasStdExtZvkb())
5386 LoV = LoSrc == 0 ? V1 : V2;
5390 HiV = HiSrc == 0 ? V1 : V2;
5396 unsigned InvRotate = NumElts - Rotation;
5406 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5429 int EvenSrc, OddSrc;
5434 int Size = Mask.size();
5436 assert(EvenSrc >= 0 &&
"Undef source?");
5437 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5441 assert(OddSrc >= 0 &&
"Undef source?");
5442 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5451 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5460 any_of(Mask, [&](
const auto &Idx) {
return Idx > 255; })) {
5489 MVT IndexContainerVT =
5494 for (
int MaskIndex : Mask) {
5495 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5504 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5514 for (
int MaskIndex : Mask) {
5515 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5516 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5518 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5533 for (
int MaskIndex : Mask) {
5534 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5538 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5570RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5572 MVT VT =
Op.getSimpleValueType();
5576 MVT ContainerVT = VT;
5579 if (
Op->isVPOpcode()) {
5580 Mask =
Op.getOperand(1);
5584 VL =
Op.getOperand(2);
5590 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 :
MVT::f32;
5592 FloatEltVT = MVT::f32;
5599 "Expected legal float type!");
5606 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5609 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5614 if (FloatVT.
bitsGT(VT)) {
5615 if (
Op->isVPOpcode())
5616 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5625 if (!
Op->isVPOpcode())
5629 MVT ContainerFloatVT =
5632 Src, Mask, RTZRM, VL);
5639 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5643 if (
Op->isVPOpcode()) {
5652 else if (IntVT.
bitsGT(VT))
5657 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5662 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5663 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5668 unsigned Adjust = ExponentBias + (EltSize - 1);
5670 if (
Op->isVPOpcode())
5680 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5681 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5691 MVT SrcVT =
Source.getSimpleValueType();
5700 SrcVT = ContainerVT;
5713 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5731 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5734 Load->getMemoryVT(),
5735 *
Load->getMemOperand()))
5739 MVT VT =
Op.getSimpleValueType();
5741 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5742 "Unexpected unaligned RVV load type");
5746 "Expecting equally-sized RVV vector types to be legal");
5748 Load->getPointerInfo(),
Load->getOriginalAlign(),
5749 Load->getMemOperand()->getFlags());
5760 assert(Store &&
Store->getValue().getValueType().isVector() &&
5761 "Expected vector store");
5764 Store->getMemoryVT(),
5765 *
Store->getMemOperand()))
5772 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5773 "Unexpected unaligned RVV store type");
5777 "Expecting equally-sized RVV vector types to be legal");
5778 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5780 Store->getPointerInfo(),
Store->getOriginalAlign(),
5781 Store->getMemOperand()->getFlags());
5786 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5815 unsigned ShiftAmt, AddOpc;
5826 MVT VT =
Op.getSimpleValueType();
5830 bool Negate =
false;
5834 if (
Index < 0 &&
Imm.isNegative()) {
5851 return DAG.
getNode(ISD::FNEG,
DL, VT, Const);
5862 if (Subtarget.hasStdExtZtso()) {
5870 return DAG.
getNode(ISD::MEMBARRIER, dl, MVT::Other,
Op.getOperand(0));
5878 return DAG.
getNode(ISD::MEMBARRIER, dl, MVT::Other,
Op.getOperand(0));
5886 MVT VT =
Op.getSimpleValueType();
5888 unsigned Check =
Op.getConstantOperandVal(1);
5889 unsigned TDCMask = 0;
5917 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5922 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5924 VL =
Op.getOperand(3);
5927 VL,
Op->getFlags());
5942 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5944 MVT MaskContainerVT =
5947 VL =
Op.getOperand(3);
5952 Mask, VL,
Op->getFlags());
5955 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5964 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5968 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5988 MVT VT =
Op.getSimpleValueType();
6015 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
6022 MVT ContainerVT = VT;
6030 if (
Op->isVPOpcode()) {
6031 Mask =
Op.getOperand(2);
6035 VL =
Op.getOperand(3);
6043 {X, X, DAG.getCondCode(ISD::SETOEQ),
6044 DAG.getUNDEF(ContainerVT), Mask, VL});
6052 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
6053 DAG.getUNDEF(ContainerVT), Mask, VL});
6059 Op.getOpcode() == ISD::FMAXIMUM ||
Op->getOpcode() == ISD::VP_FMAXIMUM
6063 DAG.
getUNDEF(ContainerVT), Mask, VL);
6071 bool IsFABS =
Op.getOpcode() == ISD::FABS;
6072 assert((IsFABS ||
Op.getOpcode() == ISD::FNEG) &&
6073 "Wrong opcode for lowering FABS or FNEG.");
6076 MVT VT =
Op.getSimpleValueType();
6077 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6084 Mask = Mask.sext(Subtarget.
getXLen());
6097 MVT VT =
Op.getSimpleValueType();
6098 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6108 if (SignSize == Subtarget.
getXLen()) {
6109 SignAsInt = DAG.
getNode(ISD::BITCAST,
DL, XLenVT, Sign);
6110 }
else if (SignSize == 16) {
6112 }
else if (SignSize == 32) {
6114 }
else if (SignSize == 64) {
6115 assert(XLenVT == MVT::i32 &&
"Unexpected type");
6125 if (ShiftAmount > 0) {
6128 }
else if (ShiftAmount < 0) {
6157#define OP_CASE(NODE) \
6159 return RISCVISD::NODE##_VL;
6160#define VP_CASE(NODE) \
6161 case ISD::VP_##NODE: \
6162 return RISCVISD::NODE##_VL;
6164 switch (
Op.getOpcode()) {
6242 case ISD::VP_CTLZ_ZERO_UNDEF:
6245 case ISD::VP_CTTZ_ZERO_UNDEF:
6254 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6259 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6264 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6267 case ISD::VP_SELECT:
6276 case ISD::VP_SIGN_EXTEND:
6278 case ISD::VP_ZERO_EXTEND:
6280 case ISD::VP_FP_TO_SINT:
6282 case ISD::VP_FP_TO_UINT:
6285 case ISD::VP_FMINNUM:
6288 case ISD::VP_FMAXNUM:
6293 case ISD::VP_LLRINT:
6305 "not a RISC-V target specific op");
6311 "adding target specific op should update this function");
6331 "not a RISC-V target specific op");
6337 "adding target specific op should update this function");
6350 if (
Op.getValueType() == MVT::nxv32f16 &&
6354 if (
Op.getValueType() == MVT::nxv32bf16)
6367 if (!
Op.getOperand(j).getValueType().isVector()) {
6368 LoOperands[j] =
Op.getOperand(j);
6369 HiOperands[j] =
Op.getOperand(j);
6372 std::tie(LoOperands[j], HiOperands[j]) =
6377 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6379 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6394 std::tie(LoOperands[j], HiOperands[j]) =
6398 if (!
Op.getOperand(j).getValueType().isVector()) {
6399 LoOperands[j] =
Op.getOperand(j);
6400 HiOperands[j] =
Op.getOperand(j);
6403 std::tie(LoOperands[j], HiOperands[j]) =
6408 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6410 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6420 auto [EVLLo, EVLHi] =
6421 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6425 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6427 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6445 if (!
Op.getOperand(j).getValueType().isVector()) {
6446 LoOperands[j] =
Op.getOperand(j);
6447 HiOperands[j] =
Op.getOperand(j);
6450 std::tie(LoOperands[j], HiOperands[j]) =
6455 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6458 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6467 switch (
Op.getOpcode()) {
6470 case ISD::ATOMIC_FENCE:
6473 return lowerGlobalAddress(
Op, DAG);
6475 return lowerBlockAddress(
Op, DAG);
6477 return lowerConstantPool(
Op, DAG);
6479 return lowerJumpTable(
Op, DAG);
6481 return lowerGlobalTLSAddress(
Op, DAG);
6485 return lowerConstantFP(
Op, DAG);
6487 return lowerSELECT(
Op, DAG);
6489 return lowerBRCOND(
Op, DAG);
6491 return lowerVASTART(
Op, DAG);
6493 return lowerFRAMEADDR(
Op, DAG);
6495 return lowerRETURNADDR(
Op, DAG);
6497 return lowerShiftLeftParts(
Op, DAG);
6499 return lowerShiftRightParts(
Op, DAG,
true);
6501 return lowerShiftRightParts(
Op, DAG,
false);
6504 if (
Op.getValueType().isFixedLengthVector()) {
6505 assert(Subtarget.hasStdExtZvkb());
6506 return lowerToScalableOp(
Op, DAG);
6508 assert(Subtarget.hasVendorXTHeadBb() &&
6509 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6510 "Unexpected custom legalization");
6515 case ISD::BITCAST: {
6517 EVT VT =
Op.getValueType();
6521 if (Op0VT == MVT::i16 &&
6523 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
6527 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6532 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6547 "Unexpected types");
6581 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6583 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6585 return LowerINTRINSIC_VOID(
Op, DAG);
6587 return LowerIS_FPCLASS(
Op, DAG);
6589 MVT VT =
Op.getSimpleValueType();
6591 assert(Subtarget.hasStdExtZvbb());
6592 return lowerToScalableOp(
Op, DAG);
6595 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6605 if (!
Op.getSimpleValueType().isVector())
6607 return lowerVectorTruncLike(
Op, DAG);
6610 if (
Op.getOperand(0).getValueType().isVector() &&
6611 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6612 return lowerVectorMaskExt(
Op, DAG, 1);
6615 if (
Op.getOperand(0).getValueType().isVector() &&
6616 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6617 return lowerVectorMaskExt(
Op, DAG, -1);
6620 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6622 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6624 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6626 MVT VT =
Op.getSimpleValueType();
6634 MVT ContainerVT = VT;
6640 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6647 MVT VT =
Op.getSimpleValueType();
6667 }
else if ((Val % 8) == 0) {
6683 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6684 Op.getOperand(1).getValueType() == MVT::i32) {
6688 DAG.
getNode(ISD::FPOWI,
DL, MVT::f32, Op0,
Op.getOperand(1));
6699 case ISD::FP_EXTEND:
6701 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6704 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6707 if (
Op.getValueType().isVector() &&
6708 ((
Op.getValueType().getScalarType() == MVT::f16 &&
6711 Op.getValueType().getScalarType() == MVT::bf16)) {
6727 Op1.getValueType().isVector() &&
6728 ((Op1.getValueType().getScalarType() == MVT::f16 &&
6731 Op1.getValueType().getScalarType() == MVT::bf16)) {
6737 Op1.getValueType().getVectorElementCount());
6740 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6750 MVT VT =
Op.getSimpleValueType();
6753 bool IsStrict =
Op->isStrictFPOpcode();
6754 SDValue Src =
Op.getOperand(0 + IsStrict);
6755 MVT SrcVT = Src.getSimpleValueType();
6766 "Unexpected vector element types");
6770 if (EltSize > (2 * SrcEltSize)) {
6782 Op.getOperand(0), Ext);
6786 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6791 auto [FExt, Chain] =
6793 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6800 if (SrcEltSize > (2 * EltSize)) {
6803 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6808 Op.getOperand(0), Src);
6823 Op.getOperand(0), Src);
6837 unsigned RVVOpc = 0;
6838 switch (
Op.getOpcode()) {
6870 "Expected same element count");
6877 Op.getOperand(0), Src, Mask, VL);
6881 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6887 case ISD::FP_TO_BF16: {
6896 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6901 case ISD::BF16_TO_FP: {
6903 MVT VT =
Op.getSimpleValueType();
6913 return DAG.
getNode(ISD::FP_EXTEND,
DL, VT, Res);
6916 case ISD::STRICT_FP_TO_FP16:
6917 case ISD::FP_TO_FP16: {
6923 bool IsStrict =
Op->isStrictFPOpcode();
6924 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
6928 std::tie(Res, Chain) =
6929 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions,
DL, Chain);
6937 case ISD::STRICT_FP16_TO_FP:
6938 case ISD::FP16_TO_FP: {
6944 bool IsStrict =
Op->isStrictFPOpcode();
6945 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
6951 std::tie(Res, Chain) =
makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
6952 CallOptions,
DL, Chain);
6960 case ISD::FNEARBYINT:
6963 case ISD::FROUNDEVEN:
6969 if (
Op.getValueType().isVector())
6973 case ISD::LLROUND: {
6974 assert(
Op.getOperand(0).getValueType() == MVT::f16 &&
6975 "Unexpected custom legalisation");
6978 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), Ext);
6984 assert(
Op.getOperand(1).getValueType() == MVT::f16 &&
6985 "Unexpected custom legalisation");
6988 {
Op.getOperand(0),
Op.getOperand(1)});
6989 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
6990 {Ext.getValue(1), Ext.getValue(0)});
6992 case ISD::VECREDUCE_ADD:
6993 case ISD::VECREDUCE_UMAX:
6994 case ISD::VECREDUCE_SMAX:
6995 case ISD::VECREDUCE_UMIN:
6996 case ISD::VECREDUCE_SMIN:
6997 return lowerVECREDUCE(
Op, DAG);
6998 case ISD::VECREDUCE_AND:
6999 case ISD::VECREDUCE_OR:
7000 case ISD::VECREDUCE_XOR:
7001 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
7002 return lowerVectorMaskVecReduction(
Op, DAG,
false);
7003 return lowerVECREDUCE(
Op, DAG);
7004 case ISD::VECREDUCE_FADD:
7005 case ISD::VECREDUCE_SEQ_FADD:
7006 case ISD::VECREDUCE_FMIN:
7007 case ISD::VECREDUCE_FMAX:
7008 case ISD::VECREDUCE_FMAXIMUM:
7009 case ISD::VECREDUCE_FMINIMUM:
7010 return lowerFPVECREDUCE(
Op, DAG);
7011 case ISD::VP_REDUCE_ADD:
7012 case ISD::VP_REDUCE_UMAX:
7013 case ISD::VP_REDUCE_SMAX:
7014 case ISD::VP_REDUCE_UMIN:
7015 case ISD::VP_REDUCE_SMIN:
7016 case ISD::VP_REDUCE_FADD:
7017 case ISD::VP_REDUCE_SEQ_FADD:
7018 case ISD::VP_REDUCE_FMIN:
7019 case ISD::VP_REDUCE_FMAX:
7020 case ISD::VP_REDUCE_FMINIMUM:
7021 case ISD::VP_REDUCE_FMAXIMUM:
7024 return lowerVPREDUCE(
Op, DAG);
7025 case ISD::VP_REDUCE_AND:
7026 case ISD::VP_REDUCE_OR:
7027 case ISD::VP_REDUCE_XOR:
7028 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
7029 return lowerVectorMaskVecReduction(
Op, DAG,
true);
7030 return lowerVPREDUCE(
Op, DAG);
7031 case ISD::VP_CTTZ_ELTS:
7032 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7033 return lowerVPCttzElements(
Op, DAG);
7037 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
7040 return lowerINSERT_SUBVECTOR(
Op, DAG);
7042 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
7044 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
7046 return lowerVECTOR_INTERLEAVE(
Op, DAG);
7048 return lowerSTEP_VECTOR(
Op, DAG);
7050 return lowerVECTOR_REVERSE(
Op, DAG);
7052 return lowerVECTOR_SPLICE(
Op, DAG);
7056 MVT VT =
Op.getSimpleValueType();
7058 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
7059 EltVT == MVT::bf16) {
7062 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
7063 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
7067 Elt = DAG.
getNode(ISD::BITCAST,
DL, MVT::i16,
Op.getOperand(0));
7069 return DAG.
getNode(ISD::BITCAST,
DL, VT,
7073 if (EltVT == MVT::i1)
7074 return lowerVectorMaskSplat(
Op, DAG);
7083 MVT VT =
Op.getSimpleValueType();
7084 MVT ContainerVT = VT;
7102 Op->ops().take_front(HalfNumOps));
7104 Op->ops().drop_front(HalfNumOps));
7108 unsigned NumOpElts =
7109 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
7112 SDValue SubVec = OpIdx.value();
7124 EVT VecTy = Load->getMemoryVT();
7131 unsigned NumElts = Sz / (NF * 8);
7132 int Log2LMUL =
Log2_64(NumElts) - 3;
7135 Flag.setNoUnsignedWrap(
true);
7137 SDValue BasePtr = Load->getBasePtr();
7145 for (
unsigned i = 0; i < NF; ++i) {
7158 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
7160 if (
Op.getValueType().isFixedLengthVector())
7161 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
7166 SDValue StoredVal = Store->getValue();
7174 unsigned NumElts = Sz / (NF * 8);
7175 int Log2LMUL =
Log2_64(NumElts) - 3;
7178 Flag.setNoUnsignedWrap(
true);
7180 SDValue Chain = Store->getChain();
7181 SDValue BasePtr = Store->getBasePtr();
7188 for (
unsigned i = 0; i < NF; ++i) {
7192 Ret = DAG.
getStore(Chain,
DL, Extract, BasePtr,
7194 Store->getOriginalAlign(),
7195 Store->getMemOperand()->getFlags());
7196 Chain = Ret.getValue(0);
7202 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
7204 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
7205 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
7210 return lowerMaskedLoad(
Op, DAG);
7213 return lowerMaskedStore(
Op, DAG);
7222 EVT VT =
Op.getValueType();
7233 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
7235 MVT VT =
Op.getSimpleValueType();
7240 "Unexpected CondCode");
7266 return DAG.
getSetCC(
DL, VT, RHS, LHS, CCVal);
7272 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
7288 return lowerToScalableOp(
Op, DAG);
7292 if (
Op.getSimpleValueType().isFixedLengthVector())
7293 return lowerToScalableOp(
Op, DAG);
7295 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
7296 "Unexpected custom legalisation");
7300 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7326 return lowerToScalableOp(
Op, DAG);
7330 EVT VT =
Op->getValueType(0);
7345 return lowerABS(
Op, DAG);
7350 if (Subtarget.hasStdExtZvbb())
7351 return lowerToScalableOp(
Op, DAG);
7353 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7355 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
7357 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7361 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
7370 return lowerToScalableOp(
Op, DAG);
7373 return lowerVectorStrictFSetcc(
Op, DAG);
7383 case ISD::VP_GATHER:
7384 return lowerMaskedGather(
Op, DAG);
7386 case ISD::VP_SCATTER:
7387 return lowerMaskedScatter(
Op, DAG);
7389 return lowerGET_ROUNDING(
Op, DAG);
7390 case ISD::SET_ROUNDING:
7391 return lowerSET_ROUNDING(
Op, DAG);
7393 return lowerEH_DWARF_CFA(
Op, DAG);
7394 case ISD::VP_SELECT:
7403 case ISD::VP_UADDSAT:
7404 case ISD::VP_USUBSAT:
7405 case ISD::VP_SADDSAT:
7406 case ISD::VP_SSUBSAT:
7408 case ISD::VP_LLRINT:
7409 return lowerVPOp(
Op, DAG);
7413 return lowerLogicVPOp(
Op, DAG);
7422 case ISD::VP_FMINNUM:
7423 case ISD::VP_FMAXNUM:
7424 case ISD::VP_FCOPYSIGN:
7431 return lowerVPOp(
Op, DAG);
7432 case ISD::VP_IS_FPCLASS:
7433 return LowerIS_FPCLASS(
Op, DAG);
7434 case ISD::VP_SIGN_EXTEND:
7435 case ISD::VP_ZERO_EXTEND:
7436 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7437 return lowerVPExtMaskOp(
Op, DAG);
7438 return lowerVPOp(
Op, DAG);
7439 case ISD::VP_TRUNCATE:
7440 return lowerVectorTruncLike(
Op, DAG);
7441 case ISD::VP_FP_EXTEND:
7442 case ISD::VP_FP_ROUND:
7443 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7444 case ISD::VP_SINT_TO_FP:
7445 case ISD::VP_UINT_TO_FP:
7446 if (
Op.getValueType().isVector() &&
7447 ((
Op.getValueType().getScalarType() == MVT::f16 &&
7450 Op.getValueType().getScalarType() == MVT::bf16)) {
7463 case ISD::VP_FP_TO_SINT:
7464 case ISD::VP_FP_TO_UINT:
7466 Op1.getValueType().isVector() &&
7467 ((Op1.getValueType().getScalarType() == MVT::f16 &&
7470 Op1.getValueType().getScalarType() == MVT::bf16)) {
7476 Op1.getValueType().getVectorElementCount());
7480 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7482 return lowerVPFPIntConvOp(
Op, DAG);
7486 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7487 return lowerVPSetCCMaskOp(
Op, DAG);
7493 case ISD::VP_BITREVERSE:
7495 return lowerVPOp(
Op, DAG);
7497 case ISD::VP_CTLZ_ZERO_UNDEF:
7498 if (Subtarget.hasStdExtZvbb())
7499 return lowerVPOp(
Op, DAG);
7500 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7502 case ISD::VP_CTTZ_ZERO_UNDEF:
7503 if (Subtarget.hasStdExtZvbb())
7504 return lowerVPOp(
Op, DAG);
7505 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7507 return lowerVPOp(
Op, DAG);
7508 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7509 return lowerVPStridedLoad(
Op, DAG);
7510 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7511 return lowerVPStridedStore(
Op, DAG);
7513 case ISD::VP_FFLOOR:
7515 case ISD::VP_FNEARBYINT:
7516 case ISD::VP_FROUND:
7517 case ISD::VP_FROUNDEVEN:
7518 case ISD::VP_FROUNDTOZERO:
7522 case ISD::VP_FMAXIMUM:
7523 case ISD::VP_FMINIMUM:
7527 case ISD::EXPERIMENTAL_VP_SPLICE:
7528 return lowerVPSpliceExperimental(
Op, DAG);
7529 case ISD::EXPERIMENTAL_VP_REVERSE:
7530 return lowerVPReverseExperimental(
Op, DAG);
7531 case ISD::EXPERIMENTAL_VP_SPLAT:
7532 return lowerVPSplatExperimental(
Op, DAG);
7535 "llvm.clear_cache only needs custom lower on Linux targets");
7538 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7539 Op.getOperand(2), Flags,
DL);
7547 MakeLibCallOptions CallOptions;
7548 std::pair<SDValue, SDValue> CallResult =
7549 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7550 {Start,
End, Flags}, CallOptions,
DL, InChain);
7553 return CallResult.second;
7570 N->getOffset(), Flags);
7599template <
class NodeTy>
7601 bool IsLocal,
bool IsExternWeak)
const {
7611 if (IsLocal && !Subtarget.allowTaggedGlobals())
7627 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
7657 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
7681 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7690 return getAddr(
N, DAG);
7697 return getAddr(
N, DAG);
7704 return getAddr(
N, DAG);
7709 bool UseGOT)
const {
7728 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
7773 Args.push_back(Entry);
7806 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7820 Addr = getStaticTLSAddr(
N, DAG,
false);
7823 Addr = getStaticTLSAddr(
N, DAG,
true);
7828 : getDynamicTLSAddr(
N, DAG);
7845 if (LHS == LHS2 && RHS == RHS2) {
7850 }
else if (LHS == RHS2 && RHS == LHS2) {
7858 return std::nullopt;
7866 MVT VT =
N->getSimpleValueType(0);
7899 if (~TrueVal == FalseVal) {
7915 if (std::optional<bool> MatchResult =
matchSetCC(LHS, RHS,
CC, TrueV)) {
7921 if (std::optional<bool> MatchResult =
matchSetCC(LHS, RHS,
CC, FalseV)) {
7939 if (Subtarget.hasShortForwardBranchOpt())
7942 unsigned SelOpNo = 0;
7952 unsigned ConstSelOpNo = 1;
7953 unsigned OtherSelOpNo = 2;
7960 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7965 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7971 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7973 std::swap(NewConstOps[0], NewConstOps[1]);
7985 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7987 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7990 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7991 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
8000 MVT VT =
Op.getSimpleValueType();
8014 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
8046 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
8048 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
8049 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
8051 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
8056 DL, VT, LHSVal, CondV);
8072 if (
Op.hasOneUse()) {
8073 unsigned UseOpc =
Op->use_begin()->getOpcode();
8082 return lowerSELECT(NewSel, DAG);
8111 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
8136 if (TrueVal - 1 == FalseVal)
8138 if (TrueVal + 1 == FalseVal)
8145 RHS == TrueV && LHS == FalseV) {
8188 LHS, RHS, TargetCC,
Op.getOperand(2));
8218 int XLenInBytes = Subtarget.
getXLen() / 8;
8220 EVT VT =
Op.getValueType();
8223 unsigned Depth =
Op.getConstantOperandVal(0);
8225 int Offset = -(XLenInBytes * 2);
8242 int XLenInBytes = Subtarget.
getXLen() / 8;
8247 EVT VT =
Op.getValueType();
8249 unsigned Depth =
Op.getConstantOperandVal(0);
8251 int Off = -XLenInBytes;
8252 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
8271 EVT VT =
Lo.getValueType();
8310 EVT VT =
Lo.getValueType();
8361 MVT VT =
Op.getSimpleValueType();
8387 MVT VecVT =
Op.getSimpleValueType();
8389 "Unexpected SPLAT_VECTOR_PARTS lowering");
8395 MVT ContainerVT = VecVT;
8415 int64_t ExtTrueVal)
const {
8417 MVT VecVT =
Op.getSimpleValueType();
8420 assert(Src.getValueType().isVector() &&
8421 Src.getValueType().getVectorElementType() == MVT::i1);
8442 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8444 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8447 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8452SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8454 MVT ExtVT =
Op.getSimpleValueType();
8458 MVT VT =
Op.getOperand(0).getSimpleValueType();
8484 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8486 EVT MaskVT =
Op.getValueType();
8489 "Unexpected type for vector mask lowering");
8491 MVT VecVT = Src.getSimpleValueType();
8495 VL =
Op.getOperand(2);
8498 MVT ContainerVT = VecVT;
8504 MVT MaskContainerVT =
8511 std::tie(Mask, VL) =
8519 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8521 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8525 DAG.
getUNDEF(ContainerVT), Mask, VL);
8536 unsigned Opc =
Op.getOpcode();
8537 bool IsVPTrunc = Opc == ISD::VP_TRUNCATE;
8540 MVT VT =
Op.getSimpleValueType();
8542 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8546 return lowerVectorMaskTruncLike(
Op, DAG);
8554 MVT SrcVT = Src.getSimpleValueType();
8559 "Unexpected vector truncate lowering");
8561 MVT ContainerVT = SrcVT;
8565 VL =
Op.getOperand(2);
8578 std::tie(Mask, VL) =
8594 }
while (SrcEltVT != DstEltVT);
8603RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8608 MVT VT =
Op.getSimpleValueType();
8609 MVT SrcVT = Src.getSimpleValueType();
8610 MVT ContainerVT = VT;
8633 Chain, Src, Mask, VL);
8634 Chain = Src.getValue(1);
8641 Chain, Src, Mask, VL);
8652RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8655 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8657 Op.getOpcode() == ISD::VP_FP_EXTEND ||
Op.getOpcode() == ISD::FP_EXTEND;
8662 MVT VT =
Op.getSimpleValueType();
8664 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8667 MVT SrcVT = Src.getSimpleValueType();
8669 bool IsDirectExtend =
8677 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8680 MVT ContainerVT = VT;
8684 VL =
Op.getOperand(2);
8698 std::tie(Mask, VL) =
8704 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8710 unsigned InterConvOpc =
8715 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8717 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8728static std::optional<MVT>
8734 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8736 if (MaxIdx < MinVLMAX)
8738 else if (MaxIdx < MinVLMAX * 2)
8740 else if (MaxIdx < MinVLMAX * 4)
8745 return std::nullopt;
8758 MVT VecVT =
Op.getSimpleValueType();
8772 MVT ContainerVT = VecVT;
8781 MVT OrigContainerVT = ContainerVT;
8785 const unsigned OrigIdx = IdxC->getZExtValue();
8788 DL, DAG, Subtarget)) {
8789 ContainerVT = *ShrunkVT;
8798 VLEN && ContainerVT.
bitsGT(M1VT)) {
8801 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8802 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8803 unsigned ExtractIdx =
8825 IsLegalInsert =
true;
8834 if (IsLegalInsert) {
8840 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8856 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8857 MVT I32ContainerVT =
8868 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8873 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8875 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8880 ValInVec, AlignedIdx);
8890 DAG.
getUNDEF(I32ContainerVT), ValLo,
8891 I32Mask, InsertI64VL);
8893 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8894 I32Mask, InsertI64VL);
8896 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8909 Idx, Mask, InsertVL, Policy);
8913 Slideup, AlignedIdx);
8928 EVT EltVT =
Op.getValueType();
8935 MVT ContainerVT = VecVT;
8951 unsigned WidenVecLen;
8954 unsigned MaxEEW = Subtarget.
getELen();
8959 "the number of elements should be power of 2");
8963 ExtractBitIdx = Idx;
8965 WideEltVT = LargestEltVT;
8968 ExtractElementIdx = DAG.
getNode(
8977 Vec = DAG.
getNode(ISD::BITCAST,
DL, WideVT, Vec);
8979 Vec, ExtractElementIdx);
8995 EltVT == MVT::bf16) {
9005 MVT ContainerVT = VecVT;
9017 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
9019 unsigned OrigIdx = IdxC->getZExtValue();
9022 unsigned RemIdx = OrigIdx % ElemsPerVReg;
9023 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
9024 unsigned ExtractIdx =
9034 std::optional<uint64_t> MaxIdx;
9038 MaxIdx = IdxC->getZExtValue();
9040 if (
auto SmallerVT =
9042 ContainerVT = *SmallerVT;
9069 DAG.
getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
9089 "Unexpected opcode");
9096 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9101 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9102 if (!
II || !
II->hasScalarOperand())
9105 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
9118 if (OpVT.
bitsLT(XLenVT)) {
9125 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9135 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
9136 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
9139 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
9150 case Intrinsic::riscv_vslide1up:
9151 case Intrinsic::riscv_vslide1down:
9152 case Intrinsic::riscv_vslide1up_mask:
9153 case Intrinsic::riscv_vslide1down_mask: {
9156 bool IsMasked = NumOps == 7;
9162 std::tie(ScalarLo, ScalarHi) =
9171 const auto [MinVLMAX, MaxVLMAX] =
9175 if (AVLInt <= MinVLMAX) {
9177 }
else if (AVLInt >= 2 * MaxVLMAX) {
9211 if (IntNo == Intrinsic::riscv_vslide1up ||
9212 IntNo == Intrinsic::riscv_vslide1up_mask) {
9214 ScalarHi, I32Mask, I32VL);
9216 ScalarLo, I32Mask, I32VL);
9219 ScalarLo, I32Mask, I32VL);
9221 ScalarHi, I32Mask, I32VL);
9270 const unsigned ElementWidth = 8;
9275 [[maybe_unused]]
unsigned MinVF =
9278 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
9282 bool Fractional = VF < LMul1VF;
9283 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
9304 MVT ContainerVT = OpVT;
9331 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9335 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9336 if (!
II || !
II->hasScalarOperand())
9339 unsigned SplatOp =
II->ScalarOperand + 1;
9352 if (OpVT.
bitsLT(XLenVT)) {
9355 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9368 EVT ValType = V.getValueType();
9369 if (ValType.isVector() && ValType.isFloatingPoint()) {
9375 if (ValType.isFixedLengthVector()) {
9377 DAG, V.getSimpleValueType(), Subtarget);
9393 unsigned IntNo =
Op.getConstantOperandVal(0);
9400 case Intrinsic::riscv_tuple_insert: {
9408 case Intrinsic::riscv_tuple_extract: {
9415 case Intrinsic::thread_pointer: {
9419 case Intrinsic::riscv_orc_b:
9420 case Intrinsic::riscv_brev8:
9421 case Intrinsic::riscv_sha256sig0:
9422 case Intrinsic::riscv_sha256sig1:
9423 case Intrinsic::riscv_sha256sum0:
9424 case Intrinsic::riscv_sha256sum1:
9425 case Intrinsic::riscv_sm3p0:
9426 case Intrinsic::riscv_sm3p1: {
9439 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9441 case Intrinsic::riscv_sm4ks:
9442 case Intrinsic::riscv_sm4ed: {
9446 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9449 case Intrinsic::riscv_zip:
9450 case Intrinsic::riscv_unzip: {
9453 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9455 case Intrinsic::riscv_mopr:
9459 case Intrinsic::riscv_moprr: {
9461 Op.getOperand(2),
Op.getOperand(3));
9463 case Intrinsic::riscv_clmul:
9466 case Intrinsic::riscv_clmulh:
9467 case Intrinsic::riscv_clmulr: {
9470 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9472 case Intrinsic::experimental_get_vector_length:
9474 case Intrinsic::experimental_cttz_elts:
9476 case Intrinsic::riscv_vmv_x_s: {
9480 case Intrinsic::riscv_vfmv_f_s:
9483 case Intrinsic::riscv_vmv_v_x:
9485 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9487 case Intrinsic::riscv_vfmv_v_f:
9489 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9490 case Intrinsic::riscv_vmv_s_x: {
9493 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9496 Op.getOperand(1), Scalar,
Op.getOperand(3));
9499 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9516 MVT VT =
Op.getSimpleValueType();
9521 if (
Op.getOperand(1).isUndef())
9537 case Intrinsic::riscv_vfmv_s_f:
9539 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9541 case Intrinsic::riscv_vaesdf_vv:
9542 case Intrinsic::riscv_vaesdf_vs:
9543 case Intrinsic::riscv_vaesdm_vv:
9544 case Intrinsic::riscv_vaesdm_vs:
9545 case Intrinsic::riscv_vaesef_vv:
9546 case Intrinsic::riscv_vaesef_vs:
9547 case Intrinsic::riscv_vaesem_vv:
9548 case Intrinsic::riscv_vaesem_vs:
9549 case Intrinsic::riscv_vaeskf1:
9550 case Intrinsic::riscv_vaeskf2:
9551 case Intrinsic::riscv_vaesz_vs:
9552 case Intrinsic::riscv_vsm4k:
9553 case Intrinsic::riscv_vsm4r_vv:
9554 case Intrinsic::riscv_vsm4r_vs: {
9555 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9556 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9557 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9562 case Intrinsic::riscv_vsm3c:
9563 case Intrinsic::riscv_vsm3me: {
9564 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9565 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9570 case Intrinsic::riscv_vsha2ch:
9571 case Intrinsic::riscv_vsha2cl:
9572 case Intrinsic::riscv_vsha2ms: {
9573 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9574 !Subtarget.hasStdExtZvknhb())
9576 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9577 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9578 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9582 case Intrinsic::riscv_sf_vc_v_x:
9583 case Intrinsic::riscv_sf_vc_v_i:
9584 case Intrinsic::riscv_sf_vc_v_xv:
9585 case Intrinsic::riscv_sf_vc_v_iv:
9586 case Intrinsic::riscv_sf_vc_v_vv:
9587 case Intrinsic::riscv_sf_vc_v_fv:
9588 case Intrinsic::riscv_sf_vc_v_xvv:
9589 case Intrinsic::riscv_sf_vc_v_ivv:
9590 case Intrinsic::riscv_sf_vc_v_vvv:
9591 case Intrinsic::riscv_sf_vc_v_fvv:
9592 case Intrinsic::riscv_sf_vc_v_xvw:
9593 case Intrinsic::riscv_sf_vc_v_ivw:
9594 case Intrinsic::riscv_sf_vc_v_vvw:
9595 case Intrinsic::riscv_sf_vc_v_fvw: {
9596 MVT VT =
Op.getSimpleValueType();
9633 MVT VT =
Op.getSimpleValueType();
9637 if (VT.isFloatingPoint()) {
9642 if (VT.isFixedLengthVector())
9652 if (VT.isFixedLengthVector())
9654 if (VT.isFloatingPoint())
9673 unsigned IntNo =
Op.getConstantOperandVal(1);
9677 case Intrinsic::riscv_seg2_load:
9678 case Intrinsic::riscv_seg3_load:
9679 case Intrinsic::riscv_seg4_load:
9680 case Intrinsic::riscv_seg5_load:
9681 case Intrinsic::riscv_seg6_load:
9682 case Intrinsic::riscv_seg7_load:
9683 case Intrinsic::riscv_seg8_load: {
9686 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9687 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9688 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9689 Intrinsic::riscv_vlseg8};
9690 unsigned NF =
Op->getNumValues() - 1;
9691 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9693 MVT VT =
Op->getSimpleValueType(0);
9713 Load->getMemoryVT(),
Load->getMemOperand());
9715 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
9724 case Intrinsic::riscv_sf_vc_v_x_se:
9726 case Intrinsic::riscv_sf_vc_v_i_se:
9728 case Intrinsic::riscv_sf_vc_v_xv_se:
9730 case Intrinsic::riscv_sf_vc_v_iv_se:
9732 case Intrinsic::riscv_sf_vc_v_vv_se:
9734 case Intrinsic::riscv_sf_vc_v_fv_se:
9736 case Intrinsic::riscv_sf_vc_v_xvv_se:
9738 case Intrinsic::riscv_sf_vc_v_ivv_se:
9740 case Intrinsic::riscv_sf_vc_v_vvv_se:
9742 case Intrinsic::riscv_sf_vc_v_fvv_se:
9744 case Intrinsic::riscv_sf_vc_v_xvw_se:
9746 case Intrinsic::riscv_sf_vc_v_ivw_se:
9748 case Intrinsic::riscv_sf_vc_v_vvw_se:
9750 case Intrinsic::riscv_sf_vc_v_fvw_se:
9759 unsigned IntNo =
Op.getConstantOperandVal(1);
9763 case Intrinsic::riscv_seg2_store:
9764 case Intrinsic::riscv_seg3_store:
9765 case Intrinsic::riscv_seg4_store:
9766 case Intrinsic::riscv_seg5_store:
9767 case Intrinsic::riscv_seg6_store:
9768 case Intrinsic::riscv_seg7_store:
9769 case Intrinsic::riscv_seg8_store: {
9772 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9773 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9774 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9775 Intrinsic::riscv_vsseg8};
9778 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9780 MVT VT =
Op->getOperand(2).getSimpleValueType();
9793 for (
unsigned i = 0; i < NF; i++)
9797 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget),
9801 FixedIntrinsic->getChain(),
9810 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9812 case Intrinsic::riscv_sf_vc_xv_se:
9814 case Intrinsic::riscv_sf_vc_iv_se:
9816 case Intrinsic::riscv_sf_vc_vv_se:
9818 case Intrinsic::riscv_sf_vc_fv_se:
9820 case Intrinsic::riscv_sf_vc_xvv_se:
9822 case Intrinsic::riscv_sf_vc_ivv_se:
9824 case Intrinsic::riscv_sf_vc_vvv_se:
9826 case Intrinsic::riscv_sf_vc_fvv_se:
9828 case Intrinsic::riscv_sf_vc_xvw_se:
9830 case Intrinsic::riscv_sf_vc_ivw_se:
9832 case Intrinsic::riscv_sf_vc_vvw_se:
9834 case Intrinsic::riscv_sf_vc_fvw_se:
9842 switch (ISDOpcode) {
9845 case ISD::VP_REDUCE_ADD:
9846 case ISD::VECREDUCE_ADD:
9848 case ISD::VP_REDUCE_UMAX:
9849 case ISD::VECREDUCE_UMAX:
9851 case ISD::VP_REDUCE_SMAX:
9852 case ISD::VECREDUCE_SMAX:
9854 case ISD::VP_REDUCE_UMIN:
9855 case ISD::VECREDUCE_UMIN:
9857 case ISD::VP_REDUCE_SMIN:
9858 case ISD::VECREDUCE_SMIN:
9860 case ISD::VP_REDUCE_AND:
9861 case ISD::VECREDUCE_AND:
9863 case ISD::VP_REDUCE_OR:
9864 case ISD::VECREDUCE_OR:
9866 case ISD::VP_REDUCE_XOR:
9867 case ISD::VECREDUCE_XOR:
9869 case ISD::VP_REDUCE_FADD:
9871 case ISD::VP_REDUCE_SEQ_FADD:
9873 case ISD::VP_REDUCE_FMAX:
9874 case ISD::VP_REDUCE_FMAXIMUM:
9876 case ISD::VP_REDUCE_FMIN:
9877 case ISD::VP_REDUCE_FMINIMUM:
9887 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9889 assert((
Op.getOpcode() == ISD::VECREDUCE_AND ||
9890 Op.getOpcode() == ISD::VECREDUCE_OR ||
9891 Op.getOpcode() == ISD::VECREDUCE_XOR ||
9892 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9893 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9894 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9895 "Unexpected reduction lowering");
9899 MVT ContainerVT = VecVT;
9908 VL =
Op.getOperand(3);
9910 std::tie(Mask, VL) =
9915 switch (
Op.getOpcode()) {
9918 case ISD::VECREDUCE_AND:
9919 case ISD::VP_REDUCE_AND: {
9927 case ISD::VECREDUCE_OR:
9928 case ISD::VP_REDUCE_OR:
9933 case ISD::VECREDUCE_XOR:
9934 case ISD::VP_REDUCE_XOR: {
9958 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9964 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9965 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9981 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9985 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9988 if (M1VT != InnerVT)
9994 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
10013 VecEVT =
Lo.getValueType();
10026 MVT ContainerVT = VecVT;
10046 Mask, VL,
DL, DAG, Subtarget);
10052static std::tuple<unsigned, SDValue, SDValue>
10056 auto Flags =
Op->getFlags();
10057 unsigned Opcode =
Op.getOpcode();
10061 case ISD::VECREDUCE_FADD: {
10067 case ISD::VECREDUCE_SEQ_FADD:
10070 case ISD::VECREDUCE_FMINIMUM:
10071 case ISD::VECREDUCE_FMAXIMUM:
10072 case ISD::VECREDUCE_FMIN:
10073 case ISD::VECREDUCE_FMAX: {
10078 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
10081 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
10089 MVT VecEltVT =
Op.getSimpleValueType();
10091 unsigned RVVOpcode;
10092 SDValue VectorVal, ScalarVal;
10093 std::tie(RVVOpcode, VectorVal, ScalarVal) =
10097 MVT ContainerVT = VecVT;
10103 MVT ResVT =
Op.getSimpleValueType();
10106 VL,
DL, DAG, Subtarget);
10107 if (
Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
10108 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
10111 if (
Op->getFlags().hasNoNaNs())
10117 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
10118 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
10124 DL, ResVT, NoNaNs, Res,
10131 unsigned Opc =
Op.getOpcode();
10154 Vec, Mask, VL,
DL, DAG, Subtarget);
10155 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
10156 Op->getFlags().hasNoNaNs())
10173 DL, ResVT, NoNaNs, Res,
10186 unsigned OrigIdx =
Op.getConstantOperandVal(2);
10189 if (OrigIdx == 0 && Vec.
isUndef())
10200 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10203 "Unexpected mask vector lowering");
10235 MVT ContainerVT = VecVT;
10242 DAG.
getUNDEF(ContainerVT), SubVec,
10259 if (OrigIdx == 0) {
10264 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
10265 SlideupAmt, Mask, VL, Policy);
10273 MVT ContainerVecVT = VecVT;
10279 MVT ContainerSubVecVT = SubVecVT;
10285 unsigned SubRegIdx;
10295 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10296 SubRegIdx = Decompose.first;
10298 (OrigIdx % Vscale));
10302 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10303 SubRegIdx = Decompose.first;
10310 bool ExactlyVecRegSized =
10312 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10327 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10331 if (SubRegIdx == RISCV::NoSubRegister) {
10354 MVT InterSubVT = ContainerVecVT;
10355 SDValue AlignedExtract = Vec;
10395 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10396 SlideupAmt, Mask, VL, Policy);
10401 if (ContainerVecVT.
bitsGT(InterSubVT))
10410 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10416 MVT SubVecVT =
Op.getSimpleValueType();
10421 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10437 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10440 "Unexpected mask vector lowering");
10474 MVT ContainerVT = VecVT;
10482 if (
auto ShrunkVT =
10484 ContainerVT = *ShrunkVT;
10497 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10509 MVT ContainerSubVecVT = SubVecVT;
10513 unsigned SubRegIdx;
10523 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10524 SubRegIdx = Decompose.first;
10526 (OrigIdx % Vscale));
10530 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10531 SubRegIdx = Decompose.first;
10558 MVT InterSubVT = VecVT;
10563 assert(SubRegIdx != RISCV::NoSubRegister);
10583 Vec, SlidedownAmt, Mask, VL);
10592 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10599 MVT VT =
N.getSimpleValueType();
10603 assert(
Op.getSimpleValueType() == VT &&
10604 "Operands and result must be same type");
10608 unsigned NumVals =
N->getNumValues();
10611 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10614 for (
unsigned I = 0;
I < NumVals;
I++) {
10620 if (TruncVals.
size() > 1)
10622 return TruncVals.
front();
10628 MVT VecVT =
Op.getSimpleValueType();
10631 "vector_interleave on non-scalable vector!");
10642 EVT SplitVT = Op0Lo.getValueType();
10645 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10647 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10661 Op.getOperand(0),
Op.getOperand(1));
10688 Concat, EvenIdx, Passthru, Mask, VL);
10690 Concat, OddIdx, Passthru, Mask, VL);
10704 MVT VecVT =
Op.getSimpleValueType();
10707 "vector_interleave on non-scalable vector!");
10720 EVT SplitVT = Op0Lo.getValueType();
10723 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10725 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10747 Op.getOperand(0),
Op.getOperand(1));
10795 MVT VT =
Op.getSimpleValueType();
10800 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10801 if (StepValImm != 1) {
10810 VL, VT,
DL, DAG, Subtarget);
10825 MVT VecVT =
Op.getSimpleValueType();
10834 MVT ContainerVT = VecVT;
10888 unsigned MaxVLMAX =
10898 if (MaxVLMAX > 256 && EltSize == 8) {
10953 DAG.
getUNDEF(ContainerVT), Mask, VL);
10965 MVT VecVT =
Op.getSimpleValueType();
10970 SDValue DownOffset, UpOffset;
10971 if (ImmValue >= 0) {
10987 DownOffset, TrueMask, UpOffset);
10988 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10994RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
11000 Load->getMemoryVT(),
11001 *
Load->getMemOperand()) &&
11002 "Expecting a correctly-aligned load");
11004 MVT VT =
Op.getSimpleValueType();
11010 const auto [MinVLMAX, MaxVLMAX] =
11013 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11027 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
11030 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11031 Ops.push_back(
Load->getBasePtr());
11036 Load->getMemoryVT(),
Load->getMemOperand());
11043RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
11049 Store->getMemoryVT(),
11050 *
Store->getMemOperand()) &&
11051 "Expecting a correctly-aligned store");
11072 const auto [MinVLMAX, MaxVLMAX] =
11075 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11086 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
11089 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
11090 Store->getMemoryVT(),
Store->getMemOperand());
11096 MVT VT =
Op.getSimpleValueType();
11099 EVT MemVT = MemSD->getMemoryVT();
11101 SDValue Chain = MemSD->getChain();
11106 Mask = VPLoad->getMask();
11108 VL = VPLoad->getVectorLength();
11111 Mask = MLoad->getMask();
11112 PassThru = MLoad->getPassThru();
11119 MVT ContainerVT = VT;
11133 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
11136 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11138 Ops.push_back(PassThru);
11139 Ops.push_back(BasePtr);
11141 Ops.push_back(Mask);
11150 Chain =
Result.getValue(1);
11163 EVT MemVT = MemSD->getMemoryVT();
11165 SDValue Chain = MemSD->getChain();
11169 bool IsCompressingStore =
false;
11171 Val = VPStore->getValue();
11172 Mask = VPStore->getMask();
11173 VL = VPStore->getVectorLength();
11176 Val = MStore->getValue();
11177 Mask = MStore->getMask();
11178 IsCompressingStore = MStore->isCompressingStore();
11187 MVT ContainerVT = VT;
11192 if (!IsUnmasked || IsCompressingStore) {
11201 if (IsCompressingStore) {
11204 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
11211 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
11213 Ops.push_back(Val);
11214 Ops.push_back(BasePtr);
11216 Ops.push_back(Mask);
11220 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11224RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
11226 MVT InVT =
Op.getOperand(0).getSimpleValueType();
11229 MVT VT =
Op.getSimpleValueType();
11250 unsigned Opc =
Op.getOpcode();
11257 MVT VT =
Op.getSimpleValueType();
11290 MVT ContainerInVT = InVT;
11309 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11313 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11321 {Chain, Op1, Op2, CC, Mask, Mask, VL});
11326 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
11339 MVT VT =
Op.getSimpleValueType();
11343 "Unexpected type for ISD::ABS");
11345 MVT ContainerVT = VT;
11352 if (
Op->getOpcode() == ISD::VP_ABS) {
11353 Mask =
Op->getOperand(1);
11357 VL =
Op->getOperand(2);
11365 DAG.
getUNDEF(ContainerVT), Mask, VL);
11367 DAG.
getUNDEF(ContainerVT), Mask, VL);
11374SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11377 MVT VT =
Op.getSimpleValueType();
11381 "Can only handle COPYSIGN with matching types.");
11390 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11395SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11397 MVT VT =
Op.getSimpleValueType();
11400 MVT I1ContainerVT =
11414 Op2, DAG.
getUNDEF(ContainerVT), VL);
11425 MVT VT =
Op.getSimpleValueType();
11430 for (
const SDValue &V :
Op->op_values()) {
11434 if (!V.getValueType().isVector()) {
11440 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
11441 "Only fixed length vectors are supported!");
11455 if (
Op->isStrictFPOpcode()) {
11464 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11478 MVT VT =
Op.getSimpleValueType();
11481 MVT ContainerVT = VT;
11490 if (HasPassthruOp) {
11493 if (*MaskIdx == OpIdx.index())
11497 if (
Op.getOpcode() == ISD::VP_MERGE) {
11501 assert(
Op.getOpcode() == ISD::VP_SELECT);
11508 if (!V.getValueType().isFixedLengthVector()) {
11513 MVT OpVT = V.getSimpleValueType();
11515 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11516 "Only fixed length vectors are supported!");
11521 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11531 MVT VT =
Op.getSimpleValueType();
11537 MVT ContainerVT = VT;
11547 DAG.
getUNDEF(ContainerVT), Zero, VL);
11550 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11552 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11555 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11564 MVT VT =
Op.getSimpleValueType();
11572 MVT ContainerVT = VT;
11582 switch (Condition) {
11650 MVT DstVT =
Op.getSimpleValueType();
11651 MVT SrcVT = Src.getSimpleValueType();
11664 if (DstEltSize >= SrcEltSize) {
11673 if (SrcEltSize == 1) {
11684 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11685 }
else if (DstEltSize > (2 * SrcEltSize)) {
11689 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11695 "Wrong input/output vector types");
11698 if (DstEltSize > (2 * SrcEltSize)) {
11714 MVT InterimFVT = DstVT;
11715 if (SrcEltSize > (2 * DstEltSize)) {
11716 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11723 if (InterimFVT != DstVT) {
11729 "Wrong input/output vector types");
11733 if (DstEltSize == 1) {
11736 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11746 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11756 while (InterimIVT != DstVT) {
11768 MVT VT =
Op.getSimpleValueType();
11775RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11787 MVT VT =
Op.getSimpleValueType();
11788 MVT ContainerVT = VT;
11798 if (IsMaskVector) {
11809 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11818 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11822 SDValue DownOffset, UpOffset;
11823 if (ImmValue >= 0) {
11837 Op1, DownOffset, Mask, UpOffset);
11841 if (IsMaskVector) {
11845 {Result, DAG.getConstant(0, DL, ContainerVT),
11846 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11861 MVT VT =
Op.getSimpleValueType();
11863 MVT ContainerVT = VT;
11879RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11882 MVT VT =
Op.getSimpleValueType();
11889 MVT ContainerVT = VT;
11897 MVT GatherVT = ContainerVT;
11901 if (IsMaskVector) {
11912 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11918 unsigned MaxVLMAX =
11927 if (MaxVLMAX > 256 && EltSize == 8) {
11955 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11957 if (IsMaskVector) {
11980 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11982 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11984 DAG.
getUNDEF(GatherVT), Mask, EVL);
11986 if (IsMaskVector) {
12001 MVT VT =
Op.getSimpleValueType();
12003 return lowerVPOp(
Op, DAG);
12010 MVT ContainerVT = VT;
12029 MVT VT =
Op.getSimpleValueType();
12030 MVT ContainerVT = VT;
12042 : Intrinsic::riscv_vlse_mask,
12045 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
12046 VPNode->getStride()};
12054 Ops.
push_back(VPNode->getVectorLength());
12062 VPNode->getMemoryVT(), VPNode->getMemOperand());
12077 SDValue StoreVal = VPNode->getValue();
12079 MVT ContainerVT = VT;
12090 : Intrinsic::riscv_vsse_mask,
12093 VPNode->getBasePtr(), VPNode->getStride()};
12101 Ops.
push_back(VPNode->getVectorLength());
12104 Ops, VPNode->getMemoryVT(),
12105 VPNode->getMemOperand());
12117 MVT VT =
Op.getSimpleValueType();
12120 EVT MemVT = MemSD->getMemoryVT();
12122 SDValue Chain = MemSD->getChain();
12129 Index = VPGN->getIndex();
12130 Mask = VPGN->getMask();
12132 VL = VPGN->getVectorLength();
12138 Index = MGN->getIndex();
12139 Mask = MGN->getMask();
12140 PassThru = MGN->getPassThru();
12144 MVT IndexVT =
Index.getSimpleValueType();
12148 "Unexpected VTs!");
12149 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12152 "Unexpected extending MGATHER/VP_GATHER");
12158 MVT ContainerVT = VT;
12182 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
12199 Chain =
Result.getValue(1);
12217 EVT MemVT = MemSD->getMemoryVT();
12219 SDValue Chain = MemSD->getChain();
12222 [[maybe_unused]]
bool IsTruncatingStore =
false;
12226 Index = VPSN->getIndex();
12227 Mask = VPSN->getMask();
12228 Val = VPSN->getValue();
12229 VL = VPSN->getVectorLength();
12231 IsTruncatingStore =
false;
12235 Index = MSN->getIndex();
12236 Mask = MSN->getMask();
12237 Val = MSN->getValue();
12238 IsTruncatingStore = MSN->isTruncatingStore();
12242 MVT IndexVT =
Index.getSimpleValueType();
12246 "Unexpected VTs!");
12247 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12250 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
12256 MVT ContainerVT = VT;
12280 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
12290 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
12299 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12307 static const int Table =
12331 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12337 static const unsigned Table =
12360 bool isRISCV64 = Subtarget.
is64Bit();
12424 switch (
N->getOpcode()) {
12426 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12432 "Unexpected custom legalisation");
12433 bool IsStrict =
N->isStrictFPOpcode();
12436 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12454 Opc,
DL, VTs, Chain, Op0,
12465 Op0 = DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f32, Op0);
12488 std::tie(Result, Chain) =
12495 case ISD::LROUND: {
12505 Op0 = DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f32, Op0);
12517 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12526 case ISD::READCYCLECOUNTER:
12527 case ISD::READSTEADYCOUNTER: {
12528 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12529 "has custom type legalization on riscv32");
12531 SDValue LoCounter, HiCounter;
12533 if (
N->getOpcode() == ISD::READCYCLECOUNTER) {
12535 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12537 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12540 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12542 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12546 N->getOperand(0), LoCounter, HiCounter);
12570 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12571 unsigned XLen = Subtarget.
getXLen();
12574 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12582 if (LHSIsU == RHSIsU)
12599 if (RHSIsU && LHSIsS && !RHSIsS)
12600 Results.push_back(MakeMULPair(LHS, RHS));
12601 else if (LHSIsU && RHSIsS && !LHSIsS)
12602 Results.push_back(MakeMULPair(RHS, LHS));
12611 "Unexpected custom legalisation");
12618 "Unexpected custom legalisation");
12621 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12647 "Unexpected custom legalisation");
12648 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12649 Subtarget.hasVendorXTHeadBb()) &&
12650 "Unexpected custom legalization");
12652 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12661 "Unexpected custom legalisation");
12675 MVT VT =
N->getSimpleValueType(0);
12676 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12677 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12678 "Unexpected custom legalisation");
12690 if (VT != MVT::i32)
12699 "Unexpected custom legalisation");
12720 EVT OType =
N->getValueType(1);
12733 "Unexpected custom legalisation");
12750 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12754 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12761 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res, LHS,
12772 !Subtarget.hasStdExtZbb() &&
"Unexpected custom legalisation");
12781 "Unexpected custom legalisation");
12787 "Unexpected custom legalisation");
12789 if (Subtarget.hasStdExtZbb()) {
12822 case ISD::BITCAST: {
12823 EVT VT =
N->getValueType(0);
12828 if (VT == MVT::i16 &&
12830 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
12833 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12838 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12840 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12861 MVT VT =
N->getSimpleValueType(0);
12863 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12864 "Unexpected custom legalisation");
12867 "Unexpected extension");
12893 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12895 "Unexpected EXTRACT_VECTOR_ELT legalization");
12898 MVT ContainerVT = VecVT;
12913 DAG.
getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
12926 DAG.
getUNDEF(ContainerVT), Mask, VL);
12934 unsigned IntNo =
N->getConstantOperandVal(0);
12938 "Don't know how to custom type legalize this intrinsic!");
12939 case Intrinsic::experimental_get_vector_length: {
12944 case Intrinsic::experimental_cttz_elts: {
12950 case Intrinsic::riscv_orc_b:
12951 case Intrinsic::riscv_brev8:
12952 case Intrinsic::riscv_sha256sig0:
12953 case Intrinsic::riscv_sha256sig1:
12954 case Intrinsic::riscv_sha256sum0:
12955 case Intrinsic::riscv_sha256sum1:
12956 case Intrinsic::riscv_sm3p0:
12957 case Intrinsic::riscv_sm3p1: {
12958 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12978 case Intrinsic::riscv_sm4ks:
12979 case Intrinsic::riscv_sm4ed: {
12991 case Intrinsic::riscv_mopr: {
12992 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13002 case Intrinsic::riscv_moprr: {
13003 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13015 case Intrinsic::riscv_clmul: {
13016 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13027 case Intrinsic::riscv_clmulh:
13028 case Intrinsic::riscv_clmulr: {
13029 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13057 case Intrinsic::riscv_vmv_x_s: {
13058 EVT VT =
N->getValueType(0);
13060 if (VT.
bitsLT(XLenVT)) {
13069 "Unexpected custom legalization");
13096 case ISD::VECREDUCE_ADD:
13097 case ISD::VECREDUCE_AND:
13098 case ISD::VECREDUCE_OR:
13099 case ISD::VECREDUCE_XOR:
13100 case ISD::VECREDUCE_SMAX:
13101 case ISD::VECREDUCE_UMAX:
13102 case ISD::VECREDUCE_SMIN:
13103 case ISD::VECREDUCE_UMIN:
13107 case ISD::VP_REDUCE_ADD:
13108 case ISD::VP_REDUCE_AND:
13109 case ISD::VP_REDUCE_OR:
13110 case ISD::VP_REDUCE_XOR:
13111 case ISD::VP_REDUCE_SMAX:
13112 case ISD::VP_REDUCE_UMAX:
13113 case ISD::VP_REDUCE_SMIN:
13114 case ISD::VP_REDUCE_UMIN:
13135 return ISD::VECREDUCE_ADD;
13137 return ISD::VECREDUCE_UMAX;
13139 return ISD::VECREDUCE_SMAX;
13141 return ISD::VECREDUCE_UMIN;
13143 return ISD::VECREDUCE_SMIN;
13145 return ISD::VECREDUCE_AND;
13147 return ISD::VECREDUCE_OR;
13149 return ISD::VECREDUCE_XOR;
13152 return ISD::VECREDUCE_FADD;
13178 const EVT VT =
N->getValueType(0);
13179 const unsigned Opc =
N->getOpcode();
13186 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
13191 "Inconsistent mappings");
13195 if (!LHS.hasOneUse() || !RHS.hasOneUse())
13223 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
13234 if (LHS.getOpcode() != ReduceOpc)
13249 auto Flags = ReduceVec->
getFlags();
13251 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
13261 auto BinOpToRVVReduce = [](
unsigned Opc) {
13290 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
13293 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
13296 unsigned Opc =
N->getOpcode();
13297 unsigned ReduceIdx;
13298 if (IsReduction(
N->getOperand(0), Opc))
13300 else if (IsReduction(
N->getOperand(1), Opc))
13306 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
13368 if (!Subtarget.hasStdExtZba())
13372 EVT VT =
N->getValueType(0);
13388 int64_t C0 = N0C->getSExtValue();
13389 int64_t C1 = N1C->getSExtValue();
13390 if (C0 <= 0 || C1 <= 0)
13394 int64_t Bits = std::min(C0, C1);
13395 int64_t Diff = std::abs(C0 - C1);
13396 if (Diff != 1 && Diff != 2 && Diff != 3)
13423 EVT VT =
N->getValueType(0);
13431 if ((!Subtarget.hasStdExtZicond() &&
13432 !Subtarget.hasVendorXVentanaCondOps()) ||
13454 bool SwapSelectOps;
13460 SwapSelectOps =
false;
13461 NonConstantVal = FalseVal;
13463 SwapSelectOps =
true;
13464 NonConstantVal = TrueVal;
13470 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13518 EVT VT =
N->getValueType(0);
13533 if (!N0C->hasOneUse())
13535 int64_t C0 = N0C->getSExtValue();
13536 int64_t C1 = N1C->getSExtValue();
13538 if (C0 == -1 || C0 == 0 || C0 == 1 ||
isInt<12>(C1))
13545 }
else if ((C1 / C0 + 1) != 0 &&
isInt<12>(C1 / C0 + 1) &&
13549 }
else if ((C1 / C0 - 1) != 0 &&
isInt<12>(C1 / C0 - 1) &&
13577 EVT VT =
N->getValueType(0);
13608 unsigned OuterExtend =
13612 OuterExtend,
SDLoc(
N), VT,
13620 EVT VT =
N->getValueType(0);
13668 EVT VT =
N->getValueType(0);
13678 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13688 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13709 if (!Subtarget.hasStdExtZbb())
13712 EVT VT =
N->getValueType(0);
13714 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
13724 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13739 EVT VT =
N->getValueType(0);
13794 EVT VT =
N->getValueType(0);
13818 EVT VT =
N->getValueType(0);
13845 if (CondLHS != True)
13852 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
13864 if (!FalseRHSC || !FalseRHSC->
isZero())
13884 EVT VT =
N->getValueType(0);
13891 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
13918 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
13970 EVT VT =
N->getValueType(0);
14016 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
14046 const APInt &Imm = ConstN00->getAPIntValue();
14047 if ((Imm + 1).isSignedIntN(12))
14068 EVT VT =
N->getValueType(0);
14080 const bool HasShlAdd =
14081 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
14095 for (
uint64_t Divisor : {3, 5, 9}) {
14096 if (MulAmt % Divisor != 0)
14098 uint64_t MulAmt2 = MulAmt / Divisor;
14106 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
14123 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
14139 if (ScaleShift >= 1 && ScaleShift < 4) {
14140 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
14154 for (
uint64_t Divisor : {3, 5, 9}) {
14159 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
14170 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
14172 if (ScaleShift >= 1 && ScaleShift < 4) {
14173 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
14199 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
14201 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
14212 for (
uint64_t Divisor : {3, 5, 9}) {
14213 if (MulAmt % Divisor != 0)
14215 uint64_t MulAmt2 = MulAmt / Divisor;
14218 for (
uint64_t Divisor2 : {3, 5, 9}) {
14219 if (MulAmt2 % Divisor2 != 0)
14221 uint64_t MulAmt3 = MulAmt2 / Divisor2;
14244 EVT VT =
N->getValueType(0);
14251 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
14252 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
14265 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
14266 V3 != (HalfSize - 1))
14276 return DAG.
getNode(ISD::BITCAST,
DL, VT, Sra);
14282 EVT VT =
N->getValueType(0);
14290 unsigned AddSubOpc;
14296 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
14297 AddSubOpc = V->getOpcode();
14309 if (IsAddSubWith1(N0)) {
14311 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
14314 if (IsAddSubWith1(N1)) {
14316 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
14331 if (isIndexTypeSigned(IndexType))
14334 if (!
N->hasOneUse())
14337 EVT VT =
N.getValueType();
14376 EVT SrcVT = Src.getValueType();
14380 NewElen = std::max(NewElen, 8U);
14403 EVT VT =
N->getValueType(0);
14406 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14422 if (!isIntEqualitySetCC(
Cond))
14431 const APInt &C1 = N1C->getAPIntValue();
14449 EVT VT =
N->getValueType(0);
14455 Subtarget.hasStdExtZfhmin())
14457 Src.getOperand(0));
14465struct CombineResult;
14467enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
14494struct NodeExtensionHelper {
14503 bool SupportsFPExt;
14506 bool EnforceOneUse;
14521 return OrigOperand;
14532 unsigned getExtOpc(ExtKind SupportsExt)
const {
14533 switch (SupportsExt) {
14534 case ExtKind::SExt:
14536 case ExtKind::ZExt:
14538 case ExtKind::FPExt:
14549 std::optional<ExtKind> SupportsExt)
const {
14550 if (!SupportsExt.has_value())
14551 return OrigOperand;
14553 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
14557 if (
Source.getValueType() == NarrowVT)
14561 if (
Source.getValueType().getVectorElementType() == MVT::bf16) {
14567 unsigned ExtOpc = getExtOpc(*SupportsExt);
14571 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14578 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14586 assert(
Source.getOpcode() == ISD::FP_EXTEND &&
"Unexpected source");
14590 DAG.
getUNDEF(NarrowVT), Source, VL);
14603 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
14609 MVT EltVT = SupportsExt == ExtKind::FPExt
14611 :
MVT::getIntegerVT(NarrowSize);
14613 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
14614 "Trying to extend something we can't represent");
14621 static unsigned getSExtOpcode(
unsigned Opcode) {
14644 static unsigned getZExtOpcode(
unsigned Opcode) {
14670 static unsigned getFPExtOpcode(
unsigned Opcode) {
14695 static unsigned getSUOpcode(
unsigned Opcode) {
14697 "SU is only supported for MUL");
14703 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14723 using CombineToTry = std::function<std::optional<CombineResult>(
14724 SDNode * ,
const NodeExtensionHelper & ,
14729 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14733 unsigned Opc = OrigOperand.
getOpcode();
14737 "Unexpected Opcode");
14750 unsigned ScalarBits =
Op.getValueSizeInBits();
14752 if (ScalarBits < EltBits) {
14755 !Subtarget.
is64Bit() &&
"Unexpected splat");
14757 SupportsSExt =
true;
14761 SupportsZExt =
true;
14763 EnforceOneUse =
false;
14767 unsigned NarrowSize = EltBits / 2;
14770 if (NarrowSize < 8)
14774 SupportsSExt =
true;
14778 SupportsZExt =
true;
14780 EnforceOneUse =
false;
14783 bool isSupportedFPExtend(
SDNode *Root,
MVT NarrowEltVT,
14790 if (NarrowEltVT == MVT::bf16 && (!Subtarget.hasStdExtZvfbfwma() ||
14800 SupportsZExt =
false;
14801 SupportsSExt =
false;
14802 SupportsFPExt =
false;
14803 EnforceOneUse =
true;
14804 unsigned Opc = OrigOperand.
getOpcode();
14826 SupportsZExt =
true;
14829 SupportsSExt =
true;
14834 if (!isSupportedFPExtend(Root, NarrowEltVT, Subtarget))
14836 SupportsFPExt =
true;
14841 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14850 if (
Op.getOpcode() != ISD::FP_EXTEND)
14853 if (!isSupportedFPExtend(Root,
Op.getOperand(0).getSimpleValueType(),
14858 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
14859 if (NarrowSize != ScalarBits)
14862 SupportsFPExt =
true;
14871 static bool isSupportedRoot(
const SDNode *Root,
14900 Subtarget.hasStdExtZvbb();
14902 return Subtarget.hasStdExtZvbb();
14916 assert(isSupportedRoot(Root, Subtarget) &&
14917 "Trying to build an helper with an "
14918 "unsupported root");
14919 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14935 if (OperandIdx == 1) {
14944 EnforceOneUse =
false;
14949 fillUpExtensionSupport(Root, DAG, Subtarget);
14955 static std::pair<SDValue, SDValue>
14958 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14977 switch (
N->getOpcode()) {
15019struct CombineResult {
15021 unsigned TargetOpcode;
15023 std::optional<ExtKind> LHSExt;
15024 std::optional<ExtKind> RHSExt;
15028 NodeExtensionHelper
LHS;
15030 NodeExtensionHelper
RHS;
15032 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
15033 const NodeExtensionHelper &LHS, std::optional<ExtKind> LHSExt,
15034 const NodeExtensionHelper &RHS, std::optional<ExtKind> RHSExt)
15035 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
15036 LHS(LHS), RHS(RHS) {}
15044 std::tie(Mask, VL) =
15045 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
15059 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
15060 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
15061 Passthru, Mask, VL);
15075static std::optional<CombineResult>
15076canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
15077 const NodeExtensionHelper &RHS,
15080 if ((AllowExtMask & ExtKind::ZExt) && LHS.SupportsZExt && RHS.SupportsZExt)
15081 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
15082 Root, LHS, {ExtKind::ZExt}, RHS,
15084 if ((AllowExtMask & ExtKind::SExt) && LHS.SupportsSExt && RHS.SupportsSExt)
15085 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
15086 Root, LHS, {ExtKind::SExt}, RHS,
15088 if ((AllowExtMask & ExtKind::FPExt) && LHS.SupportsFPExt && RHS.SupportsFPExt)
15089 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
15090 Root, LHS, {ExtKind::FPExt}, RHS,
15092 return std::nullopt;
15101static std::optional<CombineResult>
15102canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
15105 return canFoldToVWWithSameExtensionImpl(
15106 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
15114static std::optional<CombineResult>
15115canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
15118 if (RHS.SupportsFPExt)
15119 return CombineResult(
15120 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
15121 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
15128 return CombineResult(
15129 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
15130 LHS, std::nullopt, RHS, {ExtKind::ZExt});
15132 return CombineResult(
15133 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
15134 LHS, std::nullopt, RHS, {ExtKind::SExt});
15135 return std::nullopt;
15142static std::optional<CombineResult>
15143canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15146 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
15154static std::optional<CombineResult>
15155canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15158 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
15166static std::optional<CombineResult>
15167canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15170 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
15178static std::optional<CombineResult>
15179canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
15183 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
15184 return std::nullopt;
15185 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
15186 Root, LHS, {ExtKind::SExt}, RHS,
15191NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
15202 Strategies.
push_back(canFoldToVWWithSameExtension);
15211 Strategies.
push_back(canFoldToVWWithSameExtension);
15216 Strategies.
push_back(canFoldToVWWithSameExtension);
15223 Strategies.
push_back(canFoldToVWWithZEXT);
15228 Strategies.
push_back(canFoldToVWWithSEXT);
15233 Strategies.
push_back(canFoldToVWWithZEXT);
15238 Strategies.
push_back(canFoldToVWWithFPEXT);
15267 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
15273 Inserted.insert(
N);
15276 while (!Worklist.
empty()) {
15279 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
15280 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
15281 auto AppendUsersIfNeeded = [&Worklist, &Subtarget,
15282 &Inserted](
const NodeExtensionHelper &
Op) {
15283 if (
Op.needToPromoteOtherUsers()) {
15285 UE =
Op.OrigOperand->use_end();
15288 if (!NodeExtensionHelper::isSupportedRoot(TheUse, Subtarget))
15291 if (UI.getOperandNo() >= 2)
15293 if (Inserted.insert(TheUse).second)
15306 NodeExtensionHelper::getSupportedFoldings(Root);
15308 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
15309 bool Matched =
false;
15310 for (
int Attempt = 0;
15311 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
15314 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
15315 FoldingStrategies) {
15316 std::optional<CombineResult> Res =
15317 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
15324 if (Res->LHSExt.has_value())
15325 if (!AppendUsersIfNeeded(LHS))
15327 if (Res->RHSExt.has_value())
15328 if (!AppendUsersIfNeeded(RHS))
15340 SDValue InputRootReplacement;
15347 for (CombineResult Res : CombinesToApply) {
15348 SDValue NewValue = Res.materialize(DAG, Subtarget);
15349 if (!InputRootReplacement) {
15351 "First element is expected to be the current node");
15352 InputRootReplacement = NewValue;
15357 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
15361 return InputRootReplacement;
15368 unsigned Opc =
N->getOpcode();
15374 unsigned MergeOpc = MergeOp.
getOpcode();
15399 Z = Z.getOperand(1);
15405 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15412 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
15441 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
15446 if (LSNode1->
getOpcode() == ISD::LOAD) {
15449 if (MemVT == MVT::i32)
15455 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15456 {LSNode1->getChain(), BasePtr,
15457 DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
15472 {LSNode1->getChain(), LSNode1->getOperand(1), LSNode2->getOperand(1),
15473 BasePtr, DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
15490 if (!Subtarget.hasVendorXTHeadMemPair())
15495 unsigned OpNum = LSNode1->
getOpcode() == ISD::LOAD ? 1 : 2;
15502 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15505 return {
Ptr->getOperand(0), C1->getZExtValue()};
15509 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
15532 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
15535 if (Base1 != Base2)
15539 bool Valid =
false;
15540 if (MemVT == MVT::i32) {
15544 }
else if (MemVT == MVT::i64) {
15580 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15588 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15598 EVT VT =
N->getValueType(0);
15601 MVT SrcVT = Src.getSimpleValueType();
15602 MVT SrcContainerVT = SrcVT;
15604 SDValue XVal = Src.getOperand(0);
15631 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15635 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15639 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
15652 if (VT != MVT::i32 && VT != XLenVT)
15682 EVT DstVT =
N->getValueType(0);
15683 if (DstVT != XLenVT)
15689 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15697 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15709 if (SatVT == DstVT)
15711 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15717 Src = Src.getOperand(0);
15738 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15744 EVT VT =
N->getValueType(0);
15804 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15806 V.getOperand(2) == VL) {
15808 V = V.getOperand(0);
15815 bool NegA = invertIfNegative(
A);
15816 bool NegB = invertIfNegative(
B);
15817 bool NegC = invertIfNegative(
C);
15820 if (!NegA && !NegB && !NegC)
15824 if (
N->isTargetStrictFPOpcode())
15826 {N->getOperand(0), A, B, C, Mask, VL});
15840 if (
N->isTargetStrictFPOpcode())
15850 EVT VT =
N->getValueType(0);
15857 uint64_t ShAmt =
N->getConstantOperandVal(1);
15870 if (LShAmt < ExtSize) {
15883 if (ShAmt > 32 || VT != MVT::i64)
15913 U->getConstantOperandVal(1) > 32)
15968 if (!
Cond.hasOneUse())
15987 EVT VT =
Cond.getValueType();
16032 LHS = LHS.getOperand(0);
16042 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
16050 RHS = LHS.getOperand(1);
16051 LHS = LHS.getOperand(0);
16060 RHS = LHS.getOperand(1);
16061 LHS = LHS.getOperand(0);
16072 uint64_t ShAmt = LHS.getConstantOperandVal(1);
16077 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
16118 bool Commutative =
true;
16119 unsigned Opc = TrueVal.getOpcode();
16127 Commutative =
false;
16139 if (FalseVal == TrueVal.getOperand(0))
16141 else if (Commutative && FalseVal == TrueVal.getOperand(1))
16146 EVT VT =
N->getValueType(0);
16148 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
16154 assert(IdentityOperand &&
"No identity operand!");
16159 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
16160 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
16204 if (
Cond->getOperand(0) != CountZeroesArgument)
16220 CountZeroes, BitWidthMinusOne);
16230 EVT VT =
N->getValueType(0);
16231 EVT CondVT =
Cond.getValueType();
16239 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
16245 const APInt &MaskVal = LHS.getConstantOperandAPInt(1);
16266 SDValue TrueVal =
N->getOperand(1);
16267 SDValue FalseVal =
N->getOperand(2);
16282 EVT VT =
N->getValueType(0);
16289 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
16304 if (
Op.isUndef()) {
16317 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
16326 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
16354 const unsigned InVecOpcode = InVec->
getOpcode();
16371 InVecLHS, InValLHS, EltNo);
16373 InVecRHS, InValRHS, EltNo);
16374 return DAG.
getNode(InVecOpcode,
DL, VT, LHS, RHS);
16385 unsigned Elt = IndexC->getZExtValue();
16393 unsigned ConcatOpIdx = Elt / ConcatNumElts;
16396 ConcatOp, InVal, NewIdx);
16400 ConcatOps[ConcatOpIdx] = ConcatOp;
16412 EVT VT =
N->getValueType(0);
16424 !
SDValue(BaseLd, 0).hasOneUse())
16427 EVT BaseLdVT = BaseLd->getValueType(0);
16435 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
16437 Ld->getValueType(0) != BaseLdVT)
16446 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
16448 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16453 if (BIO1.equalBaseIndex(BIO2, DAG))
16458 SDValue P2 = Ld2->getBasePtr();
16464 return std::nullopt;
16468 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
16473 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
16474 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
16482 unsigned WideScalarBitWidth =
16495 auto [StrideVariant, MustNegateStride] = *BaseDiff;
16497 std::holds_alternative<SDValue>(StrideVariant)
16498 ? std::get<SDValue>(StrideVariant)
16501 if (MustNegateStride)
16510 ConstStride && ConstStride->getSExtValue() >= 0)
16514 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
16520 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16524 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16539 if (
N->getValueType(0).isFixedLengthVector())
16547 if (!AddPassthruOp.
isUndef())
16551 auto IsVWMulOpc = [](
unsigned Opc) {
16570 if (!MulPassthruOp.
isUndef())
16580 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
16581 }(
N, DAG, Subtarget);
16586 if (AddMask != MulMask || AddVL != MulVL)
16591 "Unexpected opcode after VWMACC_VL");
16593 "Unexpected opcode after VWMACC_VL!");
16595 "Unexpected opcode after VWMUL_VL!");
16597 "Unexpected opcode after VWMUL_VL!");
16600 EVT VT =
N->getValueType(0);
16616 const EVT IndexVT =
Index.getValueType();
16620 if (!isIndexTypeSigned(IndexType))
16650 assert(ShuffleMask.empty());
16652 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16655 if (
Index->getOperand(i)->isUndef())
16658 if (
C % ElementSize != 0)
16660 C =
C / ElementSize;
16663 ShuffleMask.push_back(
C);
16664 ActiveLanes.
set(
C);
16666 return ActiveLanes.
all();
16684 if (NumElems % 2 != 0)
16688 const unsigned WiderElementSize = ElementSize * 2;
16689 if (WiderElementSize > ST.getELen()/8)
16692 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16695 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16698 if (
Index->getOperand(i)->isUndef())
16704 if (
C % WiderElementSize != 0)
16709 if (
C !=
Last + ElementSize)
16729 Mask.getOperand(0) != VL)
16732 auto IsTruncNode = [&](
SDValue V) {
16734 V.getOperand(1) == Mask && V.getOperand(2) == VL;
16741 while (IsTruncNode(
Op)) {
16742 if (!
Op.hasOneUse())
16744 Op =
Op.getOperand(0);
16779 MVT VT =
N->getSimpleValueType(0);
16784 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
16786 if (V.getOpcode() != Opc &&
16787 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
16788 V.getOperand(3) == Mask && V.getOperand(4) == VL))
16796 Op.getOperand(1).getValueType().isFixedLengthVector() &&
16798 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
16800 Op =
Op.getOperand(1).getOperand(0);
16803 return V.getOperand(0);
16806 Op.getOperand(2) == VL) {
16809 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
16819 auto DetectUSatPattern = [&](
SDValue V) {
16844 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
16850 auto DetectSSatPattern = [&](
SDValue V) {
16852 unsigned NumSrcBits = V.getScalarValueSizeInBits();
16860 if (HiC == SignedMax && LoC == SignedMin)
16866 if (HiC == SignedMax && LoC == SignedMin)
16876 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
16878 Src = Src.getOperand(0);
16882 if ((Val = DetectUSatPattern(Src)))
16884 else if ((Val = DetectSSatPattern(Src)))
16894 Val = DAG.
getNode(ClipOpc,
DL, ValVT, Val, Mask, VL);
16895 }
while (ValVT != VT);
16909 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16920 switch (
N->getOpcode()) {
16940 APInt V =
C->getValueAPF().bitcastToAPInt();
16975 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16976 SimplifyDemandedLowBitsHelper(1, 5))
16984 if (SimplifyDemandedLowBitsHelper(0, 32))
17001 MVT VT =
N->getSimpleValueType(0);
17017 "Unexpected value type!");
17027 LN0->getBasePtr(), IVT, LN0->getMemOperand());
17051 EVT VT =
N->getValueType(0);
17104 if (!
C || !
C->getValueAPF().isExactlyValue(+1.0))
17106 EVT VT =
N->getValueType(0);
17120 case ISD::FMINNUM: {
17135 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
17140 Src.getOperand(0));
17145 Src.getOperand(0), Src.getOperand(1));
17166 unsigned Opc =
N->getOpcode();
17181 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
17204 EVT VT =
N->getValueType(0);
17207 if (TrueV == FalseV)
17239 {LHS, RHS, CC, TrueV, FalseV});
17306 N->getOperand(0), LHS, RHS,
CC,
N->getOperand(4));
17319 EVT VT =
N->getValueType(0);
17331 if (In2.
getOpcode() != ISD::FP_EXTEND &&
17340 DAG.
getNode(ISD::FNEG,
DL, VT, NewFPExtRound));
17342 case ISD::MGATHER: {
17344 const EVT VT =
N->getValueType(0);
17346 SDValue ScaleOp = MGN->getScale();
17348 assert(!MGN->isIndexScaled() &&
17349 "Scaled gather/scatter should not be formed");
17354 N->getVTList(), MGN->getMemoryVT(),
DL,
17355 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17356 MGN->getBasePtr(), Index, ScaleOp},
17357 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17361 N->getVTList(), MGN->getMemoryVT(),
DL,
17362 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17363 MGN->getBasePtr(), Index, ScaleOp},
17364 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17370 if (std::optional<VIDSequence> SimpleVID =
17372 SimpleVID && SimpleVID->StepDenominator == 1) {
17373 const int64_t StepNumerator = SimpleVID->StepNumerator;
17374 const int64_t Addend = SimpleVID->Addend;
17381 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
17389 VT,
DL, MGN->getChain(), BasePtr,
17391 EVL, MGN->getMemOperand());
17393 StridedLoad, MGN->getPassThru(), EVL);
17403 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
17405 MGN->getMemoryVT(), MGN->getMemOperand(),
17414 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
17416 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
17418 EVT IndexVT =
Index.getValueType()
17425 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
17427 EltCnt.divideCoefficientBy(2));
17430 EltCnt.divideCoefficientBy(2));
17435 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
17443 case ISD::MSCATTER:{
17446 SDValue ScaleOp = MSN->getScale();
17448 assert(!MSN->isIndexScaled() &&
17449 "Scaled gather/scatter should not be formed");
17454 N->getVTList(), MSN->getMemoryVT(),
DL,
17455 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17457 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17461 N->getVTList(), MSN->getMemoryVT(),
DL,
17462 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17464 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17466 EVT VT = MSN->getValue()->getValueType(0);
17468 if (!MSN->isTruncatingStore() &&
17472 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
17473 DAG.
getUNDEF(XLenVT), MSN->getMask(),
17474 MSN->getMemoryVT(), MSN->getMemOperand(),
17479 case ISD::VP_GATHER: {
17482 SDValue ScaleOp = VPGN->getScale();
17484 assert(!VPGN->isIndexScaled() &&
17485 "Scaled gather/scatter should not be formed");
17490 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17491 ScaleOp, VPGN->getMask(),
17492 VPGN->getVectorLength()},
17493 VPGN->getMemOperand(), IndexType);
17497 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17498 ScaleOp, VPGN->getMask(),
17499 VPGN->getVectorLength()},
17500 VPGN->getMemOperand(), IndexType);
17504 case ISD::VP_SCATTER: {
17507 SDValue ScaleOp = VPSN->getScale();
17509 assert(!VPSN->isIndexScaled() &&
17510 "Scaled gather/scatter should not be formed");
17515 {VPSN->getChain(), VPSN->getValue(),
17516 VPSN->getBasePtr(), Index, ScaleOp,
17517 VPSN->getMask(), VPSN->getVectorLength()},
17518 VPSN->getMemOperand(), IndexType);
17522 {VPSN->getChain(), VPSN->getValue(),
17523 VPSN->getBasePtr(), Index, ScaleOp,
17524 VPSN->getMask(), VPSN->getVectorLength()},
17525 VPSN->getMemOperand(), IndexType);
17539 EVT VT =
N->getValueType(0);
17542 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
17561 EVT VT =
N->getValueType(0);
17565 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
17602 if (
N->getOpcode() != ISD::STORE)
17606 SDValue Chain = Store->getChain();
17607 EVT MemVT = Store->getMemoryVT();
17608 SDValue Val = Store->getValue();
17611 bool IsScalarizable =
17613 Store->isSimple() &&
17643 NewVT, *Store->getMemOperand())) {
17645 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
17646 Store->getPointerInfo(), Store->getOriginalAlign(),
17647 Store->getMemOperand()->getFlags());
17657 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
17659 L->getMemoryVT() == MemVT) {
17662 NewVT, *Store->getMemOperand()) &&
17664 NewVT, *L->getMemOperand())) {
17666 L->getPointerInfo(), L->getOriginalAlign(),
17667 L->getMemOperand()->getFlags());
17668 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
17669 Store->getPointerInfo(), Store->getOriginalAlign(),
17670 Store->getMemOperand()->getFlags());
17682 MVT VecVT = Src.getSimpleValueType();
17689 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
17692 Store->getMemOperand(), Store->getAddressingMode(),
17693 Store->isTruncatingStore(),
false);
17700 EVT VT =
N->getValueType(0);
17722 const MVT VT =
N->getSimpleValueType(0);
17724 SDValue Scalar =
N->getOperand(1);
17733 const MVT VT =
N->getSimpleValueType(0);
17735 SDValue Scalar =
N->getOperand(1);
17740 unsigned ScalarSize = Scalar.getValueSizeInBits();
17742 if (ScalarSize > EltWidth && Passthru.
isUndef())
17743 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
17750 (!Const || Const->isZero() ||
17751 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17761 if (
N->getOperand(0).isUndef() &&
17764 Src.getOperand(0).getValueType().isScalableVector()) {
17765 EVT VT =
N->getValueType(0);
17766 EVT SrcVT = Src.getOperand(0).getValueType();
17770 return Src.getOperand(0);
17776 const MVT VT =
N->getSimpleValueType(0);
17778 SDValue Scalar =
N->getOperand(1);
17782 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
17783 return Scalar.getOperand(0);
17792 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
17802 Const && !Const->isZero() &&
isInt<5>(Const->getSExtValue()) &&
17810 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17812 if (M1VT.
bitsLT(VecVT)) {
17823 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17828 case Intrinsic::riscv_vcpop:
17829 case Intrinsic::riscv_vcpop_mask:
17830 case Intrinsic::riscv_vfirst:
17831 case Intrinsic::riscv_vfirst_mask: {
17833 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17834 IntNo == Intrinsic::riscv_vfirst_mask)
17840 EVT VT =
N->getValueType(0);
17841 if (IntNo == Intrinsic::riscv_vfirst ||
17842 IntNo == Intrinsic::riscv_vfirst_mask)
17848 case ISD::BITCAST: {
17851 EVT VT =
N->getValueType(0);
17855 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17874 EVT XVT,
unsigned KeptBits)
const {
17879 if (XVT != MVT::i32 && XVT != MVT::i64)
17883 if (KeptBits == 32 || KeptBits == 64)
17887 return Subtarget.hasStdExtZbb() &&
17888 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17896 "Expected shift op");
17905 if (Ty.isScalarInteger() &&
17910 const APInt &C1Int = C1->getAPIntValue();
17911 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17932 ShiftedC1Int, Ty.getSizeInBits(), Subtarget,
17937 if (C1Cost < ShiftedC1Cost)
17951 EVT VT =
Op.getValueType();
17955 unsigned Opcode =
Op.getOpcode();
17963 const APInt &Mask =
C->getAPIntValue();
17972 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17973 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17975 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17976 if (NewMask == Mask)
17981 Op.getOperand(0), NewC);
17994 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17995 if (IsLegalMask(NewMask))
17996 return UseMask(NewMask);
17999 if (VT == MVT::i64) {
18001 if (IsLegalMask(NewMask))
18002 return UseMask(NewMask);
18017 APInt NewMask = ShrunkMask;
18018 if (MinSignedBits <= 12)
18020 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
18026 assert(IsLegalMask(NewMask));
18027 return UseMask(NewMask);
18031 static const uint64_t GREVMasks[] = {
18032 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
18033 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
18035 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
18036 unsigned Shift = 1 << Stage;
18037 if (ShAmt & Shift) {
18039 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
18051 const APInt &DemandedElts,
18053 unsigned Depth)
const {
18055 unsigned Opc =
Op.getOpcode();
18060 "Should use MaskedValueIsZero if you don't know whether Op"
18061 " is a target node!");
18144 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
18147 if (MaxVLenB == MinVLenB)
18164 case Intrinsic::riscv_vsetvli:
18165 case Intrinsic::riscv_vsetvlimax: {
18166 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
18167 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
18173 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
18177 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
18179 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
18192 unsigned Depth)
const {
18193 switch (
Op.getOpcode()) {
18199 if (Tmp == 1)
return 1;
18202 return std::min(Tmp, Tmp2);
18214 if (Tmp < 33)
return 1;
18239 unsigned XLen = Subtarget.
getXLen();
18240 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
18241 if (EltBits <= XLen)
18242 return XLen - EltBits + 1;
18246 unsigned IntNo =
Op.getConstantOperandVal(1);
18250 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
18251 case Intrinsic::riscv_masked_atomicrmw_add_i64:
18252 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
18253 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
18254 case Intrinsic::riscv_masked_atomicrmw_max_i64:
18255 case Intrinsic::riscv_masked_atomicrmw_min_i64:
18256 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
18257 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
18258 case Intrinsic::riscv_masked_cmpxchg_i64:
18266 assert(Subtarget.hasStdExtA());
18281 switch (
Op.getOpcode()) {
18287 return !
Op.getValueType().isInteger();
18295 assert(Ld &&
"Unexpected null LoadSDNode");
18304 if (!CNode || CNode->isMachineConstantPoolEntry() ||
18305 CNode->getOffset() != 0)
18313 auto *CNode = GetSupportedConstantPool(
Ptr);
18314 if (!CNode || CNode->getTargetFlags() != 0)
18317 return CNode->getConstVal();
18325 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
18326 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
18332 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
18335 return CNodeLo->getConstVal();
18340 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
18372 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18375 int64_t LoCounter =
MI.getOperand(2).getImm();
18376 int64_t HiCounter =
MI.getOperand(3).getImm();
18386 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
18398 MI.eraseFromParent();
18406 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
18414 Register SrcReg =
MI.getOperand(2).getReg();
18434 MI.eraseFromParent();
18441 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
18442 "Unexpected instruction");
18448 Register DstReg =
MI.getOperand(0).getReg();
18471 MI.eraseFromParent();
18476 switch (
MI.getOpcode()) {
18479 case RISCV::Select_GPR_Using_CC_GPR:
18480 case RISCV::Select_GPR_Using_CC_Imm:
18481 case RISCV::Select_FPR16_Using_CC_GPR:
18482 case RISCV::Select_FPR16INX_Using_CC_GPR:
18483 case RISCV::Select_FPR32_Using_CC_GPR:
18484 case RISCV::Select_FPR32INX_Using_CC_GPR:
18485 case RISCV::Select_FPR64_Using_CC_GPR:
18486 case RISCV::Select_FPR64INX_Using_CC_GPR:
18487 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18493 unsigned RelOpcode,
unsigned EqOpcode,
18496 Register DstReg =
MI.getOperand(0).getReg();
18497 Register Src1Reg =
MI.getOperand(1).getReg();
18498 Register Src2Reg =
MI.getOperand(2).getReg();
18500 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18524 MI.eraseFromParent();
18575 F->insert(It, FirstMBB);
18576 F->insert(It, SecondMBB);
18577 F->insert(It, SinkMBB);
18626 First.eraseFromParent();
18665 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
18666 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
18667 Next != BB->
end() &&
Next->getOpcode() ==
MI.getOpcode() &&
18668 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
18669 Next->getOperand(5).isKill())
18674 if (
MI.getOperand(2).isReg())
18675 RHS =
MI.getOperand(2).getReg();
18680 SelectDests.
insert(
MI.getOperand(0).getReg());
18684 SequenceMBBI != E; ++SequenceMBBI) {
18685 if (SequenceMBBI->isDebugInstr())
18688 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
18689 !SequenceMBBI->getOperand(2).isReg() ||
18690 SequenceMBBI->getOperand(2).getReg() != RHS ||
18691 SequenceMBBI->getOperand(3).getImm() !=
CC ||
18692 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
18693 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
18695 LastSelectPseudo = &*SequenceMBBI;
18697 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
18700 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18701 SequenceMBBI->mayLoadOrStore() ||
18702 SequenceMBBI->usesCustomInsertionHook())
18705 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
18721 F->insert(
I, TailMBB);
18724 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
18730 TailMBB->
push_back(DebugInstr->removeFromParent());
18734 TailMBB->
splice(TailMBB->
end(), HeadMBB,
18744 if (
MI.getOperand(2).isImm())
18747 .
addImm(
MI.getOperand(2).getImm())
18759 auto SelectMBBI =
MI.getIterator();
18760 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
18761 auto InsertionPoint = TailMBB->
begin();
18762 while (SelectMBBI != SelectEnd) {
18763 auto Next = std::next(SelectMBBI);
18766 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18767 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18768 .
addReg(SelectMBBI->getOperand(4).getReg())
18770 .
addReg(SelectMBBI->getOperand(5).getReg())
18785 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18786 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18788 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18789 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18795 unsigned CVTXOpc) {
18801 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18814 .
add(
MI.getOperand(1))
18815 .
add(
MI.getOperand(2))
18816 .
add(
MI.getOperand(3))
18818 .
add(
MI.getOperand(4))
18819 .
add(
MI.getOperand(5))
18820 .
add(
MI.getOperand(6))
18835 .
add(
MI.getOperand(0))
18836 .
add(
MI.getOperand(1))
18838 .
add(
MI.getOperand(3))
18840 .
add(
MI.getOperand(4))
18841 .
add(
MI.getOperand(5))
18842 .
add(
MI.getOperand(6))
18852 MI.eraseFromParent();
18858 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18860 switch (
MI.getOpcode()) {
18863 case RISCV::PseudoFROUND_H:
18864 CmpOpc = RISCV::FLT_H;
18865 F2IOpc = RISCV::FCVT_W_H;
18866 I2FOpc = RISCV::FCVT_H_W;
18867 FSGNJOpc = RISCV::FSGNJ_H;
18868 FSGNJXOpc = RISCV::FSGNJX_H;
18869 RC = &RISCV::FPR16RegClass;
18871 case RISCV::PseudoFROUND_H_INX:
18872 CmpOpc = RISCV::FLT_H_INX;
18873 F2IOpc = RISCV::FCVT_W_H_INX;
18874 I2FOpc = RISCV::FCVT_H_W_INX;
18875 FSGNJOpc = RISCV::FSGNJ_H_INX;
18876 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18877 RC = &RISCV::GPRF16RegClass;
18879 case RISCV::PseudoFROUND_S:
18880 CmpOpc = RISCV::FLT_S;
18881 F2IOpc = RISCV::FCVT_W_S;
18882 I2FOpc = RISCV::FCVT_S_W;
18883 FSGNJOpc = RISCV::FSGNJ_S;
18884 FSGNJXOpc = RISCV::FSGNJX_S;
18885 RC = &RISCV::FPR32RegClass;
18887 case RISCV::PseudoFROUND_S_INX:
18888 CmpOpc = RISCV::FLT_S_INX;
18889 F2IOpc = RISCV::FCVT_W_S_INX;
18890 I2FOpc = RISCV::FCVT_S_W_INX;
18891 FSGNJOpc = RISCV::FSGNJ_S_INX;
18892 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18893 RC = &RISCV::GPRF32RegClass;
18895 case RISCV::PseudoFROUND_D:
18897 CmpOpc = RISCV::FLT_D;
18898 F2IOpc = RISCV::FCVT_L_D;
18899 I2FOpc = RISCV::FCVT_D_L;
18900 FSGNJOpc = RISCV::FSGNJ_D;
18901 FSGNJXOpc = RISCV::FSGNJX_D;
18902 RC = &RISCV::FPR64RegClass;
18904 case RISCV::PseudoFROUND_D_INX:
18906 CmpOpc = RISCV::FLT_D_INX;
18907 F2IOpc = RISCV::FCVT_L_D_INX;
18908 I2FOpc = RISCV::FCVT_D_L_INX;
18909 FSGNJOpc = RISCV::FSGNJ_D_INX;
18910 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18911 RC = &RISCV::GPRRegClass;
18924 F->insert(
I, DoneMBB);
18935 Register DstReg =
MI.getOperand(0).getReg();
18936 Register SrcReg =
MI.getOperand(1).getReg();
18937 Register MaxReg =
MI.getOperand(2).getReg();
18938 int64_t FRM =
MI.getOperand(3).getImm();
18943 Register FabsReg =
MRI.createVirtualRegister(RC);
18947 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18962 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18984 MI.eraseFromParent();
18991 switch (
MI.getOpcode()) {
18994 case RISCV::ReadCounterWide:
18996 "ReadCounterWide is only to be used on riscv32");
18998 case RISCV::Select_GPR_Using_CC_GPR:
18999 case RISCV::Select_GPR_Using_CC_Imm:
19000 case RISCV::Select_FPR16_Using_CC_GPR:
19001 case RISCV::Select_FPR16INX_Using_CC_GPR:
19002 case RISCV::Select_FPR32_Using_CC_GPR:
19003 case RISCV::Select_FPR32INX_Using_CC_GPR:
19004 case RISCV::Select_FPR64_Using_CC_GPR:
19005 case RISCV::Select_FPR64INX_Using_CC_GPR:
19006 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
19008 case RISCV::BuildPairF64Pseudo:
19010 case RISCV::SplitF64Pseudo:
19012 case RISCV::PseudoQuietFLE_H:
19014 case RISCV::PseudoQuietFLE_H_INX:
19015 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
19016 case RISCV::PseudoQuietFLT_H:
19018 case RISCV::PseudoQuietFLT_H_INX:
19019 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
19020 case RISCV::PseudoQuietFLE_S:
19022 case RISCV::PseudoQuietFLE_S_INX:
19023 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
19024 case RISCV::PseudoQuietFLT_S:
19026 case RISCV::PseudoQuietFLT_S_INX:
19027 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
19028 case RISCV::PseudoQuietFLE_D:
19030 case RISCV::PseudoQuietFLE_D_INX:
19031 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
19032 case RISCV::PseudoQuietFLE_D_IN32X:
19035 case RISCV::PseudoQuietFLT_D:
19037 case RISCV::PseudoQuietFLT_D_INX:
19038 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
19039 case RISCV::PseudoQuietFLT_D_IN32X:
19043 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
19045 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
19047 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
19049 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
19051 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
19053 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
19055 case RISCV::PseudoFROUND_H:
19056 case RISCV::PseudoFROUND_H_INX:
19057 case RISCV::PseudoFROUND_S:
19058 case RISCV::PseudoFROUND_S_INX:
19059 case RISCV::PseudoFROUND_D:
19060 case RISCV::PseudoFROUND_D_INX:
19061 case RISCV::PseudoFROUND_D_IN32X:
19063 case TargetOpcode::STATEPOINT:
19069 MI.addOperand(*
MI.getMF(),
19075 case TargetOpcode::STACKMAP:
19076 case TargetOpcode::PATCHPOINT:
19079 "supported on 64-bit targets");
19097 if (
MI.readsRegister(RISCV::FRM,
nullptr))
19103void RISCVTargetLowering::analyzeInputArgs(
19107 unsigned NumArgs = Ins.size();
19110 for (
unsigned i = 0; i != NumArgs; ++i) {
19111 MVT ArgVT = Ins[i].VT;
19114 Type *ArgTy =
nullptr;
19117 else if (Ins[i].isOrigArg())
19118 ArgTy = FType->
getParamType(Ins[i].getOrigArgIndex());
19121 true, IsRet, ArgTy)) {
19122 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19129void RISCVTargetLowering::analyzeOutputArgs(
19133 unsigned NumArgs = Outs.
size();
19135 for (
unsigned i = 0; i != NumArgs; i++) {
19136 MVT ArgVT = Outs[i].VT;
19138 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19141 Outs[i].IsFixed, IsRet, OrigTy)) {
19142 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19193 if (In.isOrigArg()) {
19198 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19199 (
BitWidth < 32 && In.Flags.isZExt())) {
19221 if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32)
19234 Val = DAG.
getNode(ISD::BITCAST,
DL, LocVT, Val);
19270 ExtType,
DL, LocVT, Chain, FIN,
19287 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19300 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19309 SDValue Chain, CallingConv::ID CallConv,
bool IsVarArg,
19315 switch (CallConv) {
19325 if (Subtarget.hasStdExtE())
19329 "(Zdinx/D) instruction set extensions");
19333 if (Func.hasFnAttribute(
"interrupt")) {
19334 if (!Func.arg_empty())
19336 "Functions with the interrupt attribute cannot have arguments!");
19341 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19343 "Function interrupt attribute argument not supported!");
19348 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19350 std::vector<SDValue> OutChains;
19359 analyzeInputArgs(MF, CCInfo, Ins,
false,
19363 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19384 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19385 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19387 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19389 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19418 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() - Idx);
19423 if (VarArgsSaveSize == 0) {
19427 int VaArgOffset = -VarArgsSaveSize;
19435 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19436 VarArgsSaveSize += XLenInBytes;
19443 for (
unsigned I = Idx;
I < ArgRegs.
size(); ++
I) {
19444 const Register Reg = RegInfo.createVirtualRegister(RC);
19445 RegInfo.addLiveIn(ArgRegs[
I], Reg);
19448 Chain,
DL, ArgValue, FIN,
19450 OutChains.push_back(Store);
19464 if (!OutChains.empty()) {
19465 OutChains.push_back(Chain);
19475bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19479 auto CalleeCC = CLI.CallConv;
19480 auto &Outs = CLI.Outs;
19482 auto CallerCC = Caller.getCallingConv();
19489 if (Caller.hasFnAttribute(
"interrupt"))
19504 for (
auto &VA : ArgLocs)
19510 auto IsCallerStructRet = Caller.hasStructRetAttr();
19511 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19512 if (IsCallerStructRet || IsCalleeStructRet)
19517 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19518 if (CalleeCC != CallerCC) {
19519 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19520 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19527 for (
auto &Arg : Outs)
19528 if (Arg.Flags.isByVal())
19551 CallingConv::ID CallConv = CLI.
CallConv;
19563 if (Subtarget.hasStdExtE())
19567 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19573 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19579 "site marked musttail");
19586 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19588 if (!Flags.isByVal())
19592 unsigned Size = Flags.getByValSize();
19593 Align Alignment = Flags.getNonZeroByValAlign();
19600 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19602 false,
nullptr, IsTailCall,
19614 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19617 SDValue ArgValue = OutVals[OutIdx];
19637 if (!StackPtr.getNode())
19644 Chain,
DL,
Hi, Address,
19649 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19667 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19668 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19674 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19675 SDValue PartValue = OutVals[OutIdx + 1];
19676 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19690 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19692 for (
const auto &Part : Parts) {
19693 SDValue PartValue = Part.first;
19694 SDValue PartOffset = Part.second;
19701 ArgValue = SpillSlot;
19707 if (Flags.isByVal())
19708 ArgValue = ByValArgs[j++];
19715 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19716 "for passing parameters");
19719 if (!StackPtr.getNode())
19733 if (!MemOpChains.
empty())
19739 for (
auto &Reg : RegsToPass) {
19740 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19747 validateCCReservedRegs(RegsToPass, MF);
19752 "Return address register required, but has been reserved."});
19757 bool CalleeIsLargeExternalSymbol =
false;
19763 CalleeIsLargeExternalSymbol =
true;
19779 for (
auto &Reg : RegsToPass)
19785 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19786 assert(Mask &&
"Missing call preserved mask for calling convention");
19795 "Unexpected CFI type for a direct call");
19803 bool NeedSWGuarded =
false;
19805 Subtarget.hasStdExtZicfilp() &&
19807 NeedSWGuarded =
true;
19821 Chain = DAG.
getNode(CallOpc,
DL, NodeTys, Ops);
19834 analyzeInputArgs(MF, RetCCInfo, Ins,
true,
CC_RISCV);
19837 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19838 auto &VA = RVLocs[i];
19846 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19847 assert(VA.needsCustom());
19867 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
19869 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19870 MVT VT = Outs[i].VT;
19873 true,
true,
nullptr))
19905 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19906 SDValue Val = OutVals[OutIdx];
19915 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19919 Register RegHi = RVLocs[++i].getLocReg();
19925 "Return value register required, but has been reserved."});
19941 "Return value register required, but has been reserved."});
19963 if (Func.hasFnAttribute(
"interrupt")) {
19964 if (!Func.getReturnType()->isVoidTy())
19966 "Functions with the interrupt attribute must have void return type!");
19972 if (Kind ==
"supervisor")
19978 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19981void RISCVTargetLowering::validateCCReservedRegs(
19982 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19991 F,
"Argument register required, but has been reserved."});
19997 if (
N->getNumValues() != 1)
19999 if (!
N->hasNUsesOfValue(1, 0))
20002 SDNode *Copy = *
N->use_begin();
20004 if (Copy->getOpcode() == ISD::BITCAST) {
20016 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20020 bool HasRet =
false;
20021 for (
SDNode *Node : Copy->uses()) {
20029 Chain = Copy->getOperand(0);
20038#define NODE_NAME_CASE(NODE) \
20039 case RISCVISD::NODE: \
20040 return "RISCVISD::" #NODE;
20302#undef NODE_NAME_CASE
20309 if (Constraint.
size() == 1) {
20310 switch (Constraint[0]) {
20326 if (Constraint ==
"vr" || Constraint ==
"vm")
20332std::pair<unsigned, const TargetRegisterClass *>
20338 if (Constraint.
size() == 1) {
20339 switch (Constraint[0]) {
20344 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20345 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20346 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20347 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20348 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20349 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20350 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20352 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20353 return std::make_pair(0U, &RISCV::FPR16RegClass);
20354 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20355 return std::make_pair(0U, &RISCV::FPR32RegClass);
20356 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20357 return std::make_pair(0U, &RISCV::FPR64RegClass);
20362 }
else if (Constraint ==
"vr") {
20363 for (
const auto *RC :
20364 {&RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
20365 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN3M1RegClass,
20366 &RISCV::VRN4M1RegClass, &RISCV::VRN5M1RegClass,
20367 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass,
20368 &RISCV::VRN8M1RegClass, &RISCV::VRN2M2RegClass,
20369 &RISCV::VRN3M2RegClass, &RISCV::VRN4M2RegClass,
20370 &RISCV::VRN2M4RegClass}) {
20372 return std::make_pair(0U, RC);
20374 }
else if (Constraint ==
"vm") {
20375 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20376 return std::make_pair(0U, &RISCV::VMV0RegClass);
20384 .
Case(
"{zero}", RISCV::X0)
20385 .
Case(
"{ra}", RISCV::X1)
20386 .
Case(
"{sp}", RISCV::X2)
20387 .
Case(
"{gp}", RISCV::X3)
20388 .
Case(
"{tp}", RISCV::X4)
20389 .
Case(
"{t0}", RISCV::X5)
20390 .
Case(
"{t1}", RISCV::X6)
20391 .
Case(
"{t2}", RISCV::X7)
20392 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20393 .
Case(
"{s1}", RISCV::X9)
20394 .
Case(
"{a0}", RISCV::X10)
20395 .
Case(
"{a1}", RISCV::X11)
20396 .
Case(
"{a2}", RISCV::X12)
20397 .
Case(
"{a3}", RISCV::X13)
20398 .
Case(
"{a4}", RISCV::X14)
20399 .
Case(
"{a5}", RISCV::X15)
20400 .
Case(
"{a6}", RISCV::X16)
20401 .
Case(
"{a7}", RISCV::X17)
20402 .
Case(
"{s2}", RISCV::X18)
20403 .
Case(
"{s3}", RISCV::X19)
20404 .
Case(
"{s4}", RISCV::X20)
20405 .
Case(
"{s5}", RISCV::X21)
20406 .
Case(
"{s6}", RISCV::X22)
20407 .
Case(
"{s7}", RISCV::X23)
20408 .
Case(
"{s8}", RISCV::X24)
20409 .
Case(
"{s9}", RISCV::X25)
20410 .
Case(
"{s10}", RISCV::X26)
20411 .
Case(
"{s11}", RISCV::X27)
20412 .
Case(
"{t3}", RISCV::X28)
20413 .
Case(
"{t4}", RISCV::X29)
20414 .
Case(
"{t5}", RISCV::X30)
20415 .
Case(
"{t6}", RISCV::X31)
20417 if (XRegFromAlias != RISCV::NoRegister)
20418 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20427 if (Subtarget.hasStdExtF()) {
20429 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20430 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20431 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20432 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20433 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20434 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20435 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20436 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20437 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20438 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20439 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20440 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20441 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20442 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20443 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20444 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20445 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20446 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20447 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20448 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20449 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20450 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20451 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20452 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20453 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20454 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20455 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20456 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20457 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20458 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20459 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20460 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20462 if (FReg != RISCV::NoRegister) {
20463 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20464 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20465 unsigned RegNo = FReg - RISCV::F0_F;
20466 unsigned DReg = RISCV::F0_D + RegNo;
20467 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20469 if (VT == MVT::f32 || VT == MVT::Other)
20470 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20471 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20472 unsigned RegNo = FReg - RISCV::F0_F;
20473 unsigned HReg = RISCV::F0_H + RegNo;
20474 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20481 .
Case(
"{v0}", RISCV::V0)
20482 .
Case(
"{v1}", RISCV::V1)
20483 .
Case(
"{v2}", RISCV::V2)
20484 .
Case(
"{v3}", RISCV::V3)
20485 .
Case(
"{v4}", RISCV::V4)
20486 .
Case(
"{v5}", RISCV::V5)
20487 .
Case(
"{v6}", RISCV::V6)
20488 .
Case(
"{v7}", RISCV::V7)
20489 .
Case(
"{v8}", RISCV::V8)
20490 .
Case(
"{v9}", RISCV::V9)
20491 .
Case(
"{v10}", RISCV::V10)
20492 .
Case(
"{v11}", RISCV::V11)
20493 .
Case(
"{v12}", RISCV::V12)
20494 .
Case(
"{v13}", RISCV::V13)
20495 .
Case(
"{v14}", RISCV::V14)
20496 .
Case(
"{v15}", RISCV::V15)
20497 .
Case(
"{v16}", RISCV::V16)
20498 .
Case(
"{v17}", RISCV::V17)
20499 .
Case(
"{v18}", RISCV::V18)
20500 .
Case(
"{v19}", RISCV::V19)
20501 .
Case(
"{v20}", RISCV::V20)
20502 .
Case(
"{v21}", RISCV::V21)
20503 .
Case(
"{v22}", RISCV::V22)
20504 .
Case(
"{v23}", RISCV::V23)
20505 .
Case(
"{v24}", RISCV::V24)
20506 .
Case(
"{v25}", RISCV::V25)
20507 .
Case(
"{v26}", RISCV::V26)
20508 .
Case(
"{v27}", RISCV::V27)
20509 .
Case(
"{v28}", RISCV::V28)
20510 .
Case(
"{v29}", RISCV::V29)
20511 .
Case(
"{v30}", RISCV::V30)
20512 .
Case(
"{v31}", RISCV::V31)
20514 if (VReg != RISCV::NoRegister) {
20515 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20516 return std::make_pair(VReg, &RISCV::VMRegClass);
20517 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20518 return std::make_pair(VReg, &RISCV::VRRegClass);
20519 for (
const auto *RC :
20520 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20521 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20522 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20523 return std::make_pair(VReg, RC);
20529 std::pair<Register, const TargetRegisterClass *> Res =
20535 if (Res.second == &RISCV::GPRF16RegClass ||
20536 Res.second == &RISCV::GPRF32RegClass ||
20537 Res.second == &RISCV::GPRPairRegClass)
20538 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20546 if (ConstraintCode.
size() == 1) {
20547 switch (ConstraintCode[0]) {
20562 if (Constraint.
size() == 1) {
20563 switch (Constraint[0]) {
20601 if (Subtarget.hasStdExtZtso()) {
20603 return Builder.CreateFence(Ord);
20608 return Builder.CreateFence(Ord);
20617 if (Subtarget.hasStdExtZtso()) {
20619 return Builder.CreateFence(Ord);
20625 if (Subtarget.enableTrailingSeqCstFence() &&
isa<StoreInst>(Inst) &&
20644 if (Subtarget.hasForcedAtomics())
20649 if (Subtarget.hasStdExtZacas() &&
20650 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20656 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20669 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20671 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20673 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20675 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20677 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20679 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20681 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20683 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20692 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20694 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20696 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20698 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20700 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20702 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20704 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20706 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20725 Builder.CreateNot(Mask,
"Inv_Mask"),
20732 unsigned XLen = Subtarget.
getXLen();
20741 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
20742 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
20743 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
20756 unsigned ValWidth =
20759 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
20760 Result = Builder.CreateCall(LrwOpScwLoop,
20761 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20764 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20768 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
20776 if (Subtarget.hasForcedAtomics())
20780 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20789 unsigned XLen = Subtarget.
getXLen();
20790 Value *Ordering = Builder.getIntN(XLen,
static_cast<uint64_t>(Ord));
20791 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20793 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
20794 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
20795 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
20796 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20801 Value *Result = Builder.CreateCall(
20802 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20804 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
20809 EVT DataVT)
const {
20825 return Subtarget.hasStdExtZfhmin();
20827 return Subtarget.hasStdExtF();
20829 return Subtarget.hasStdExtD();
20861 "RVVBitsPerBlock changed, audit needed");
20870 if (!Subtarget.hasVendorXTHeadMemIdx())
20876 Base =
Op->getOperand(0);
20878 int64_t RHSC = RHS->getSExtValue();
20884 bool isLegalIndexedOffset =
false;
20885 for (
unsigned i = 0; i < 4; i++)
20886 if (
isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20887 isLegalIndexedOffset =
true;
20891 if (!isLegalIndexedOffset)
20908 VT = LD->getMemoryVT();
20909 Ptr = LD->getBasePtr();
20911 VT = ST->getMemoryVT();
20912 Ptr = ST->getBasePtr();
20928 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
20933 Base = LS->getBasePtr();
20937 if (
Base ==
Op->getOperand(0))
20939 else if (
Base ==
Op->getOperand(1))
20951 VT = LD->getMemoryVT();
20952 Ptr = LD->getBasePtr();
20954 VT = ST->getMemoryVT();
20955 Ptr = ST->getBasePtr();
20998 const Constant *PersonalityFn)
const {
21003 const Constant *PersonalityFn)
const {
21032 const bool HasZmmul = Subtarget.hasStdExtZmmul();
21037 const APInt &Imm = ConstNode->getAPIntValue();
21040 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21041 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21045 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
21046 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21047 (Imm - 8).isPowerOf2()))
21052 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
21053 ConstNode->hasOneUse()) {
21054 APInt ImmS = Imm.ashr(Imm.countr_zero());
21055 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21056 (1 - ImmS).isPowerOf2())
21079 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
21088 unsigned *
Fast)
const {
21091 *
Fast = Subtarget.enableUnalignedScalarMem();
21092 return Subtarget.enableUnalignedScalarMem();
21108 *
Fast = Subtarget.enableUnalignedVectorMem();
21109 return Subtarget.enableUnalignedVectorMem();
21118 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21130 if (
Op.size() < MinVLenInBytes)
21145 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21149 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21151 if (
Op.isFixedDstAlign())
21152 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21154 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21162 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21163 bool IsABIRegCopy =
CC.has_value();
21165 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21166 PartVT == MVT::f32) {
21169 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::i16, Val);
21173 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::f32, Val);
21184 if (PartVTBitSize % ValueVTBitSize == 0) {
21185 assert(PartVTBitSize >= ValueVTBitSize);
21192 if (ValueEltVT != PartEltVT) {
21193 if (PartVTBitSize > ValueVTBitSize) {
21195 assert(
Count != 0 &&
"The number of element should not be zero.");
21196 EVT SameEltTypeVT =
21202 Val = DAG.
getNode(ISD::BITCAST,
DL, PartVT, Val);
21215 [[maybe_unused]]
unsigned ValLMUL =
21218 [[maybe_unused]]
unsigned PartLMUL =
21220 assert(ValNF == PartNF && ValLMUL == PartLMUL &&
21221 "RISC-V vector tuple type only accepts same register class type "
21234 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21235 bool IsABIRegCopy =
CC.has_value();
21236 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21237 PartVT == MVT::f32) {
21241 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::i32, Val);
21243 Val = DAG.
getNode(ISD::BITCAST,
DL, ValueVT, Val);
21254 if (PartVTBitSize % ValueVTBitSize == 0) {
21255 assert(PartVTBitSize >= ValueVTBitSize);
21256 EVT SameEltTypeVT = ValueVT;
21263 if (ValueEltVT != PartEltVT) {
21265 assert(
Count != 0 &&
"The number of element should not be zero.");
21268 Val = DAG.
getNode(ISD::BITCAST,
DL, SameEltTypeVT, Val);
21282 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21289 unsigned Opc =
N->getOpcode();
21319 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21338 if (FVTy->getNumElements() < 2)
21353 return Factor * LMUL <= 8;
21357 Align Alignment)
const {
21369 if (!Subtarget.enableUnalignedVectorMem() &&
21377 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21378 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21379 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21380 Intrinsic::riscv_seg8_load};
21409 {VTy, LI->getPointerOperandType(), XLenTy});
21411 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21416 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21417 Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
21418 Shuffles[i]->replaceAllUsesWith(SubVec);
21425 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21426 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21427 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21428 Intrinsic::riscv_seg8_store};
21448 unsigned Factor)
const {
21453 ShuffleVTy->getNumElements() / Factor);
21455 SI->getPointerAddressSpace(),
21456 SI->getDataLayout()))
21463 {VTy, SI->getPointerOperandType(), XLenTy});
21468 for (
unsigned i = 0; i < Factor; i++) {
21469 Value *Shuffle = Builder.CreateShuffleVector(
21477 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21478 Ops.
append({SI->getPointerOperand(), VL});
21480 Builder.CreateCall(VssegNFunc, Ops);
21495 const unsigned Factor = 2;
21510 {ResVTy, LI->getPointerOperandType(), XLenTy});
21511 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21515 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21516 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21517 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21518 Intrinsic::riscv_vlseg8};
21525 NumElts * SEW / 8),
21529 LI->
getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
21532 Value *Vlseg = Builder.CreateCall(
21534 ConstantInt::get(XLenTy,
Log2_64(SEW))});
21539 LI->
getModule(), Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy});
21540 for (
unsigned i = 0; i < Factor; ++i) {
21541 Value *VecExtract =
21542 Builder.CreateCall(VecExtractFunc, {Vlseg, Builder.getInt32(i)});
21543 Return = Builder.CreateInsertValue(Return, VecExtract, i);
21559 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
21562 const unsigned Factor = 2;
21568 SI->getPointerAddressSpace(),
DL))
21576 {InVTy, SI->getPointerOperandType(), XLenTy});
21577 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21578 Builder.CreateCall(VssegNFunc, {
II->getArgOperand(0),
II->getArgOperand(1),
21579 SI->getPointerOperand(), VL});
21582 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21583 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21584 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21585 Intrinsic::riscv_vsseg8};
21590 SI->getContext(),
"riscv.vector.tuple",
21592 NumElts * SEW / 8),
21596 SI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
21601 SI->getModule(), Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy});
21603 for (
unsigned i = 0; i < Factor; ++i)
21605 Builder.CreateCall(VecInsertFunc, {StoredVal, II->getArgOperand(i),
21606 Builder.getInt32(i)});
21608 Builder.CreateCall(VssegNFunc, {StoredVal, SI->getPointerOperand(), VL,
21609 ConstantInt::get(XLenTy,
Log2_64(SEW))});
21620 "Invalid call instruction for a KCFI check");
21622 MBBI->getOpcode()));
21625 Target.setIsRenamable(
false);
21633#define GET_REGISTER_MATCHER
21634#include "RISCVGenAsmMatcher.inc"
21640 if (Reg == RISCV::NoRegister)
21642 if (Reg == RISCV::NoRegister)
21654 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21656 if (NontemporalInfo ==
nullptr)
21664 int NontemporalLevel = 5;
21665 const MDNode *RISCVNontemporalInfo =
21666 I.getMetadata(
"riscv-nontemporal-domain");
21667 if (RISCVNontemporalInfo !=
nullptr)
21674 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21675 "RISC-V target doesn't support this non-temporal domain.");
21677 NontemporalLevel -= 2;
21679 if (NontemporalLevel & 0b1)
21681 if (NontemporalLevel & 0b10)
21694 return TargetFlags;
21704 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21707 return Subtarget.hasStdExtZbb() &&
21720 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21721 Op == Instruction::And ||
Op == Instruction::Or ||
21722 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21723 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
21724 Op == Instruction::Freeze ||
Op == Instruction::Store)
21736 if (AI->getAllocatedType()->isScalableTy())
21744RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21752 if (!Subtarget.hasShortForwardBranchOpt())
21754 EVT VT =
N->getValueType(0);
21755 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21759 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21764bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21765 EVT VT,
const APInt &AndMask)
const {
21766 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21767 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21771unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21779 if (Subtarget.hasStdExtZicfilp()) {
21791#define GET_RISCVVIntrinsicsTable_IMPL
21792#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
Promote Memory to Register
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static bool hasPassthruOp(unsigned Opcode)
Return true if a RISC-V target specified op has a passthru operand.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< APInt > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static unsigned getPACKOpcode(unsigned DestBW, const RISCVSubtarget &Subtarget)
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineOp_VLToVWOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary or FMA operation to its equivalent VW or VW_W form.
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isPromotedOpNeedingSplit(SDValue Op, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue performVFMADD_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Double the element size of the build vector to reduce the number of vslide1down in the build vector c...
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
const MachineOperand & RHS
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(T *V)
Returns the type of the given value/instruction V.
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
void clearAllBits()
Set every bit to 0.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
This class holds the attributes for a function, its return value, and its parameters.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
BasicBlock * GetInsertBlock() const
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
bool isRISCVVectorTuple() const
Return true if this is a RISCV vector tuple type where the runtime length is machine dependent.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
static MVT getRISCVVectorTupleVT(unsigned Sz, unsigned NFields)
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
unsigned getRISCVVectorTupleNumFields() const
Given a RISC-V vector tuple type, return the num_fields.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A RISCV-specific constant pool value.
static RISCVConstantPoolValue * Create(const GlobalValue *GV)
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
bool hasVInstructionsBF16Minimal() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
static RISCVII::VLMUL getLMUL(MVT VT)
int getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
bool isTargetStrictFPOpcode() const
Test if this node has a target-specific opcode that may raise FP exceptions (in the <target>ISD names...
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
static ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getRegister(Register Reg, EVT VT)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
static TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool EnableExtLdPromotion
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getInt8Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isStructTy() const
True if this is an instance of StructType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isTargetExtTy() const
Return true if this is a target extension type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
Type * getElementType() const
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ TRUNCATE_SSAT_S
TRUNCATE_[SU]SAT_[SU] - Truncate for saturated operand [SU] located in middle, prefix for SAT means i...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Create or insert an LLVM Function declaration for an intrinsic, and return it.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ TRUNCATE_VECTOR_VL_USAT
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ TRUNCATE_VECTOR_VL_SSAT
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionAddr VTableAddr Count
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool RISCVCCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
RISCVCCAssignFn - This target-specific function extends the default CCValAssign with additional infor...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getRISCVVectorTupleNumFields() const
Given a RISCV vector tuple type, return the num_fields.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
void intersectWith(const SDNodeFlags Flags)
Clear any flags in this flag set that aren't also set in Flags.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)